diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 1e0551cff..f0f01a0e9 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -129,7 +129,9 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle "openshift-etcd", "openshift-apiserver", ) - clusterInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "") + // OCPBUGS-59626: Use cluster-level informers only, don't watch all namespaces + // Remove empty namespace ("") parameter to prevent watching ALL namespaces + clusterInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient) configInformers := configv1informers.NewSharedInformerFactory(configClient, 10*time.Minute) operatorClient, dynamicInformersForAllNamespaces, err := genericoperatorclient.NewStaticPodOperatorClient( diff --git a/test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json b/test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json new file mode 100644 index 000000000..1cd1d8bc7 --- /dev/null +++ b/test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json @@ -0,0 +1,57 @@ +[ + { + "name": "[Jira:kube-apiserver][sig-api-machinery][FeatureGate:EventTTL] Event TTL Configuration should configure and validate eventTTLMinutes=5m [Timeout:90m][Serial][Disruptive][Slow][Suite:openshift/cluster-kube-apiserver-operator/conformance/serial]", + "labels": { + "Lifecycle:informing": {} + }, + "tags": { + "timeout": "90m" + }, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "informing", + "environmentSelector": {} + }, + { + "name": "[Jira:kube-apiserver][sig-api-machinery][FeatureGate:EventTTL] Event TTL Configuration should configure and validate eventTTLMinutes=10m [Timeout:90m][Serial][Disruptive][Slow][Suite:openshift/cluster-kube-apiserver-operator/conformance/serial]", + "labels": { + "Lifecycle:informing": {} + }, + "tags": { + "timeout": "90m" + }, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "informing", + "environmentSelector": {} + }, + { + "name": "[Jira:kube-apiserver][sig-api-machinery][FeatureGate:EventTTL] Event TTL Configuration should configure and validate eventTTLMinutes=15m [Timeout:90m][Serial][Disruptive][Slow][Suite:openshift/cluster-kube-apiserver-operator/conformance/serial]", + "labels": { + "Lifecycle:informing": {} + }, + "tags": { + "timeout": "90m" + }, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "informing", + "environmentSelector": {} + }, + { + "name": "[Jira:kube-apiserver][sig-api-machinery] sanity test should always pass [Suite:openshift/cluster-kube-apiserver-operator/conformance/parallel]", + "labels": {}, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "blocking", + "environmentSelector": {} + } +] diff --git a/test/extended/tests-extension/compute_kms.go b/test/extended/tests-extension/compute_kms.go new file mode 100644 index 000000000..11b7b9c22 --- /dev/null +++ b/test/extended/tests-extension/compute_kms.go @@ -0,0 +1,113 @@ +package extended + +import ( + "context" + "fmt" + "os" + "strings" + + g "github.com/onsi/ginkgo/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// YamlKmsTestCase represents a KMS test case from YAML +type YamlKmsTestCase struct { + Name string `yaml:"name"` + Initial string `yaml:"initial"` + Expected string `yaml:"expected,omitempty"` + ExpectedError string `yaml:"expectedError,omitempty"` +} + +// ComputeNode interface to handle compute nodes across different cloud platforms +type ComputeNode interface { + GetName() string + GetInstanceID() (string, error) + CreateKMSKey() string + DeleteKMSKey(keyArn string) + LoadKMSTestCasesFromYAML() ([]YamlKmsTestCase, error) + GetIamRoleNameFromId() string + RenderKmsKeyPolicy() string + UpdateKmsPolicy(keyID string) + GetRegionFromARN(arn string) string + VerifyEncryptionType(ctx context.Context, client dynamic.Interface) (string, bool) + VerifySecretEncryption(ctx context.Context, namespace, secretName string) (bool, string) + VerifyOAuthTokenEncryption(ctx context.Context, tokenType, tokenName string) (bool, string) + ExecuteCommand(command string) (string, error) +} + +// instance is the base struct for all compute node implementations +type instance struct { + nodeName string + kubeClient *kubernetes.Clientset + dynamicClient dynamic.Interface + ctx context.Context +} + +func (i *instance) GetName() string { + return i.nodeName +} + +// ExecuteCommand executes a command on the node via oc debug +func (i *instance) ExecuteCommand(command string) (string, error) { + // Use the executeNodeCommand wrapper from util.go + return executeNodeCommand(i.nodeName, command) +} + +// ComputeNodes handles a collection of ComputeNode interfaces +type ComputeNodes []ComputeNode + +// GetNodes gets master nodes according to platform with the specified label +func GetNodes(ctx context.Context, kubeClient *kubernetes.Clientset, dynamicClient dynamic.Interface, label string) (ComputeNodes, func()) { + platform := checkPlatform(kubeClient) + + switch platform { + case "aws": + return GetAwsNodes(ctx, kubeClient, dynamicClient, label) + case "gcp": + g.Skip("GCP platform KMS support not yet implemented") + return nil, nil + case "azure": + g.Skip("Azure platform KMS support not yet implemented") + return nil, nil + default: + g.Skip(fmt.Sprintf("Platform %s is not supported for KMS tests. Expected AWS, GCP, or Azure.", platform)) + return nil, nil + } +} + +// checkPlatform determines the cloud platform of the cluster +func checkPlatform(kubeClient *kubernetes.Clientset) string { + // Check for AWS-specific labels or annotations + nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{Limit: 1}) + if err != nil || len(nodes.Items) == 0 { + return "unknown" + } + + node := nodes.Items[0] + + // Check provider ID format + if providerID := node.Spec.ProviderID; providerID != "" { + if strings.HasPrefix(providerID, "aws://") { + return "aws" + } + if strings.HasPrefix(providerID, "gce://") { + return "gcp" + } + if strings.HasPrefix(providerID, "azure://") { + return "azure" + } + } + + return "unknown" +} + +// getAWSRegion gets the AWS region from environment or config +func getAWSRegion() string { + if region := os.Getenv("AWS_REGION"); region != "" { + return region + } + // Default to us-east-1 if not specified + return "us-east-1" +} diff --git a/test/extended/tests-extension/compute_kms_aws.go b/test/extended/tests-extension/compute_kms_aws.go new file mode 100644 index 000000000..46d2b77cc --- /dev/null +++ b/test/extended/tests-extension/compute_kms_aws.go @@ -0,0 +1,524 @@ +package extended + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/kms" + kmsTypes "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" + rgttypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "gopkg.in/yaml.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// awsInstance implements ComputeNode interface for AWS platform +type awsInstance struct { + instance + awsConfig aws.Config + region string +} + +// GetAwsNodes gets AWS nodes and loads cloud credentials with the specified label +func GetAwsNodes(ctx context.Context, kubeClient *kubernetes.Clientset, dynamicClient dynamic.Interface, label string) ([]ComputeNode, func()) { + // Get AWS credentials from cluster + err := getAwsCredentialFromCluster() + o.Expect(err).NotTo(o.HaveOccurred()) + + region := getAWSRegion() + + // Load AWS config + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Get node names + nodeList, err := kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("node-role.kubernetes.io/%s", label), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(nodeList.Items)).To(o.BeNumerically(">", 0), "No nodes found with label %s", label) + + var results []ComputeNode + for _, node := range nodeList.Items { + results = append(results, newAwsInstance(ctx, kubeClient, dynamicClient, node.Name, cfg, region)) + } + + return results, nil +} + +func newAwsInstance(ctx context.Context, kubeClient *kubernetes.Clientset, dynamicClient dynamic.Interface, nodeName string, awsConfig aws.Config, region string) *awsInstance { + return &awsInstance{ + instance: instance{ + nodeName: nodeName, + kubeClient: kubeClient, + dynamicClient: dynamicClient, + ctx: ctx, + }, + awsConfig: awsConfig, + region: region, + } +} + +// GetInstanceID retrieves the EC2 instance ID from the node's provider ID +func (a *awsInstance) GetInstanceID() (string, error) { + node, err := a.kubeClient.CoreV1().Nodes().Get(a.ctx, a.nodeName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get node %s: %w", a.nodeName, err) + } + + // Provider ID format: aws://// + // Example: aws:///us-east-1a/i-1234567890abcdef0 + providerID := node.Spec.ProviderID + if providerID == "" { + return "", fmt.Errorf("node %s has no provider ID", a.nodeName) + } + + parts := strings.Split(providerID, "/") + if len(parts) < 2 { + return "", fmt.Errorf("invalid provider ID format: %s", providerID) + } + + instanceID := parts[len(parts)-1] + return instanceID, nil +} + +// CreateKMSKey creates or retrieves an AWS KMS key for testing +func (a *awsInstance) CreateKMSKey() string { + Logf("[AWS-KMS] Initializing AWS KMS client in region: %s", a.region) + kmsClient := kms.NewFromConfig(a.awsConfig) + rgtClient := resourcegroupstaggingapi.NewFromConfig(a.awsConfig) + + // Check for existing test keys with the specific tag + Logf("[AWS-KMS] Searching for existing KMS keys with tag Purpose=ocp-kms-qe-ci-test") + getResourcesInput := &resourcegroupstaggingapi.GetResourcesInput{ + ResourceTypeFilters: []string{"kms"}, + TagFilters: []rgttypes.TagFilter{ + { + Key: aws.String("Purpose"), + Values: []string{"ocp-kms-qe-ci-test"}, + }, + }, + } + + existingKeys, err := rgtClient.GetResources(a.ctx, getResourcesInput) + o.Expect(err).NotTo(o.HaveOccurred()) + + var myKmsKeyArn string + + if len(existingKeys.ResourceTagMappingList) > 0 { + myKmsKeyArn = *existingKeys.ResourceTagMappingList[0].ResourceARN + Logf("[AWS-KMS] Found existing KMS key: %s", myKmsKeyArn) + g.By(fmt.Sprintf("Found existing KMS key: %s", myKmsKeyArn)) + + // Check if key is scheduled for deletion and cancel if needed + Logf("[AWS-KMS] Checking key status for: %s", myKmsKeyArn) + describeInput := &kms.DescribeKeyInput{ + KeyId: aws.String(myKmsKeyArn), + } + keyMetadata, err := kmsClient.DescribeKey(a.ctx, describeInput) + o.Expect(err).NotTo(o.HaveOccurred()) + + Logf("[AWS-KMS] Key state: %s", keyMetadata.KeyMetadata.KeyState) + if keyMetadata.KeyMetadata.DeletionDate != nil { + Logf("[AWS-KMS] Key is scheduled for deletion on: %v", keyMetadata.KeyMetadata.DeletionDate) + g.By("Canceling scheduled deletion and enabling key") + + Logf("[AWS-KMS] Canceling key deletion...") + _, err = kmsClient.CancelKeyDeletion(a.ctx, &kms.CancelKeyDeletionInput{ + KeyId: aws.String(myKmsKeyArn), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[AWS-KMS] ✓ Deletion canceled") + + Logf("[AWS-KMS] Enabling key...") + _, err = kmsClient.EnableKey(a.ctx, &kms.EnableKeyInput{ + KeyId: aws.String(myKmsKeyArn), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[AWS-KMS] ✓ Key enabled") + } else { + Logf("[AWS-KMS] Key is active and ready to use") + } + } else { + Logf("[AWS-KMS] No existing key found, creating new KMS key") + g.By("Creating new KMS key") + createKeyInput := &kms.CreateKeyInput{ + Description: aws.String("OCP KMS QE CI Test Key"), + KeySpec: kmsTypes.KeySpecSymmetricDefault, + KeyUsage: kmsTypes.KeyUsageTypeEncryptDecrypt, + Tags: []kmsTypes.Tag{ + { + TagKey: aws.String("Purpose"), + TagValue: aws.String("ocp-kms-qe-ci-test"), + }, + }, + } + + Logf("[AWS-KMS] Creating KMS key with spec: SYMMETRIC_DEFAULT, usage: ENCRYPT_DECRYPT") + createResult, err := kmsClient.CreateKey(a.ctx, createKeyInput) + if err != nil { + if strings.Contains(err.Error(), "AccessDeniedException") { + Logf("[AWS-KMS] ✗ Access denied - insufficient permissions") + g.Skip("AWS credentials don't have permission to create KMS keys") + } + Logf("[AWS-KMS] ✗ Failed to create key: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + myKmsKeyArn = *createResult.KeyMetadata.Arn + Logf("[AWS-KMS] ✓ Created new KMS key: %s", myKmsKeyArn) + Logf("[AWS-KMS] Key ID: %s", *createResult.KeyMetadata.KeyId) + g.By(fmt.Sprintf("Created KMS key: %s", myKmsKeyArn)) + } + + return myKmsKeyArn +} + +// DeleteKMSKey schedules a KMS key for deletion +func (a *awsInstance) DeleteKMSKey(keyArn string) { + Logf("[AWS-KMS] Scheduling KMS key for deletion: %s", keyArn) + kmsClient := kms.NewFromConfig(a.awsConfig) + + // Schedule key deletion with minimum waiting period (7 days) + input := &kms.ScheduleKeyDeletionInput{ + KeyId: aws.String(keyArn), + PendingWindowInDays: aws.Int32(7), // Minimum allowed by AWS + } + + result, err := kmsClient.ScheduleKeyDeletion(a.ctx, input) + if err != nil { + // Don't fail the test if key deletion fails + Logf("[AWS-KMS] Warning: Failed to schedule key deletion: %v", err) + return + } + + if result.DeletionDate != nil { + Logf("[AWS-KMS] ✓ Key scheduled for deletion on: %v", *result.DeletionDate) + } else { + Logf("[AWS-KMS] ✓ Key deletion scheduled") + } + g.By(fmt.Sprintf("Scheduled KMS key deletion: %s", keyArn)) +} + +// LoadKMSTestCasesFromYAML loads test cases from the YAML file +func (a *awsInstance) LoadKMSTestCasesFromYAML() ([]YamlKmsTestCase, error) { + testDataFile := filepath.Join("testdata", "kms_tests_aws.yaml") + + data, err := os.ReadFile(testDataFile) + if err != nil { + return nil, fmt.Errorf("failed to read test data file: %w", err) + } + + var testCases []YamlKmsTestCase + err = yaml.Unmarshal(data, &testCases) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal test cases: %w", err) + } + + return testCases, nil +} + +// GetIamRoleNameFromId retrieves the IAM role name attached to the EC2 instance +func (a *awsInstance) GetIamRoleNameFromId() string { + Logf("[AWS-IAM] Retrieving IAM role for instance: %s", a.nodeName) + instanceID, err := a.GetInstanceID() + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[AWS-IAM] Instance ID: %s", instanceID) + + ec2Client := ec2.NewFromConfig(a.awsConfig) + iamClient := iam.NewFromConfig(a.awsConfig) + + // Describe the instance to get IAM instance profile + Logf("[AWS-IAM] Describing EC2 instance...") + describeInput := &ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + } + + result, err := ec2Client.DescribeInstances(a.ctx, describeInput) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(result.Reservations)).To(o.BeNumerically(">", 0)) + o.Expect(len(result.Reservations[0].Instances)).To(o.BeNumerically(">", 0)) + + instance := result.Reservations[0].Instances[0] + o.Expect(instance.IamInstanceProfile).NotTo(o.BeNil()) + o.Expect(instance.IamInstanceProfile.Arn).NotTo(o.BeNil()) + + Logf("[AWS-IAM] Instance profile ARN: %s", *instance.IamInstanceProfile.Arn) + + // Extract profile name from ARN + arnParts := strings.Split(*instance.IamInstanceProfile.Arn, "/") + o.Expect(len(arnParts)).To(o.BeNumerically(">=", 2)) + profileName := arnParts[1] + Logf("[AWS-IAM] Instance profile name: %s", profileName) + + // Get instance profile to retrieve role + Logf("[AWS-IAM] Fetching instance profile details...") + profileInput := &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(profileName), + } + + profileOutput, err := iamClient.GetInstanceProfile(a.ctx, profileInput) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(profileOutput.InstanceProfile.Roles)).To(o.BeNumerically(">", 0)) + + roleName := *profileOutput.InstanceProfile.Roles[0].RoleName + Logf("[AWS-IAM] ✓ IAM role name: %s", roleName) + g.By(fmt.Sprintf("IAM Role Name for instance %s: %s", instanceID, roleName)) + + return roleName +} + +const keyAWSPolicyTemplate = ` +{ + "Id": "key-policy-01", + "Statement": [ + { + "Sid": "Enable IAM User Permissions", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{.AccountID}}:root" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow use of the key", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{.AccountID}}:role/{{.MasterRoleName}}" + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ], + "Resource": "*" + }, + { + "Sid": "Allow attachment of persistent resources", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{.AccountID}}:role/{{.MasterRoleName}}" + }, + "Action": [ + "kms:CreateGrant", + "kms:ListGrants", + "kms:RevokeGrant" + ], + "Resource": "*", + "Condition": { + "Bool": { + "kms:GrantIsForAWSResource": "true" + } + } + } + ] +} +` + +type KeyAWSPolicyData struct { + AccountID string + MasterRoleName string +} + +// RenderKmsKeyPolicy renders the KMS key policy template +func (a *awsInstance) RenderKmsKeyPolicy() string { + Logf("[AWS-Policy] Rendering KMS key policy template") + stsClient := sts.NewFromConfig(a.awsConfig) + + // Get AWS account ID + Logf("[AWS-Policy] Retrieving AWS account ID via STS...") + callerIdentity, err := stsClient.GetCallerIdentity(a.ctx, &sts.GetCallerIdentityInput{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + accountID := *callerIdentity.Account + Logf("[AWS-Policy] AWS Account ID: %s", accountID) + + masterRoleName := a.GetIamRoleNameFromId() + + Logf("[AWS-Policy] Parsing policy template...") + tmpl, err := template.New("keyPolicy").Parse(keyAWSPolicyTemplate) + o.Expect(err).NotTo(o.HaveOccurred()) + + var rendered bytes.Buffer + err = tmpl.Execute(&rendered, KeyAWSPolicyData{ + AccountID: accountID, + MasterRoleName: masterRoleName, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + Logf("[AWS-Policy] ✓ Policy rendered for account %s and role %s", accountID, masterRoleName) + g.By(fmt.Sprintf("Rendered KMS Policy for account %s and role %s", accountID, masterRoleName)) + return rendered.String() +} + +// UpdateKmsPolicy updates the KMS key policy +func (a *awsInstance) UpdateKmsPolicy(keyID string) { + Logf("[AWS-KMS] Updating KMS key policy for: %s", keyID) + kmsClient := kms.NewFromConfig(a.awsConfig) + kmsPolicy := a.RenderKmsKeyPolicy() + + Logf("[AWS-KMS] Applying policy to key...") + putPolicyInput := &kms.PutKeyPolicyInput{ + KeyId: aws.String(keyID), + PolicyName: aws.String("default"), + Policy: aws.String(kmsPolicy), + } + + _, err := kmsClient.PutKeyPolicy(a.ctx, putPolicyInput) + if err != nil { + Logf("[AWS-KMS] ✗ Failed to update policy: %v", err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + + Logf("[AWS-KMS] ✓ Policy updated successfully") + g.By(fmt.Sprintf("Updated KMS key policy for key: %s", keyID)) +} + +// GetRegionFromARN extracts the region from an AWS KMS ARN +// ARN format: arn:aws:kms:region:account:key/key-id +func (a *awsInstance) GetRegionFromARN(arn string) string { + parts := strings.Split(arn, ":") + if len(parts) < 4 { + Logf("[AWS] Warning: Invalid ARN format: %s", arn) + return a.region // fallback to instance region + } + return parts[3] +} + +// VerifyEncryptionType calls the generic utility function +func (a *awsInstance) VerifyEncryptionType(ctx context.Context, client dynamic.Interface) (string, bool) { + return verifyEncryptionType(ctx, client) +} + +// VerifySecretEncryption verifies that a secret is encrypted in etcd with the expected format +// Returns: (isEncrypted, encryptionFormat) +func (a *awsInstance) VerifySecretEncryption(ctx context.Context, namespace, secretName string) (bool, string) { + Logf("[Verify-Secret] Checking encryption for secret %s/%s", namespace, secretName) + + // Execute etcdctl command to get the secret from etcd + etcdKey := fmt.Sprintf("/kubernetes.io/secrets/%s/%s", namespace, secretName) + + // Use single quotes around the etcd key to prevent shell expansion + command := fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --prefix --keys-only", + etcdKey, + ) + + output, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-Secret] Failed to query etcd: %v", err) + return false, "" + } + + // Check if key exists + if !strings.Contains(output, etcdKey) { + Logf("[Verify-Secret] Secret not found in etcd") + return false, "" + } + + // Get the actual value to check encryption format + command = fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --print-value-only | head -c 20", + etcdKey, + ) + + value, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-Secret] Failed to get secret value from etcd: %v", err) + return false, "" + } + + // Check for KMSv2 encryption prefix + if strings.HasPrefix(value, "k8s:enc:kms:v2:") { + Logf("[Verify-Secret] ✓ Secret is encrypted with KMSv2") + return true, "k8s:enc:kms:v2:" + } else if strings.HasPrefix(value, "k8s:enc:kms:v1:") { + Logf("[Verify-Secret] Secret is encrypted with KMSv1") + return true, "k8s:enc:kms:v1:" + } else if strings.HasPrefix(value, "k8s:enc:") { + Logf("[Verify-Secret] Secret is encrypted with format: %s", value[:15]) + return true, value[:15] + } + + Logf("[Verify-Secret] Secret is not encrypted (no k8s:enc: prefix)") + return false, "" +} + +// VerifyOAuthTokenEncryption verifies that an OAuth token is encrypted in etcd +// Returns: (isEncrypted, encryptionFormat) +func (a *awsInstance) VerifyOAuthTokenEncryption(ctx context.Context, tokenType, tokenName string) (bool, string) { + Logf("[Verify-OAuth] Checking encryption for %s: %s", tokenType, tokenName) + + // etcd key format for OAuth tokens + var etcdKey string + if tokenType == "oauthaccesstokens" { + etcdKey = fmt.Sprintf("/kubernetes.io/oauth.openshift.io/oauthaccesstokens/%s", tokenName) + } else if tokenType == "oauthauthorizetokens" { + etcdKey = fmt.Sprintf("/kubernetes.io/oauth.openshift.io/oauthauthorizetokens/%s", tokenName) + } else { + Logf("[Verify-OAuth] Unknown token type: %s", tokenType) + return false, "" + } + + // Use single quotes around the etcd key to prevent shell expansion of special chars like ~ + command := fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --prefix --keys-only", + etcdKey, + ) + + output, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-OAuth] Failed to query etcd: %v", err) + return false, "" + } + + // Check if key exists + if !strings.Contains(output, etcdKey) { + Logf("[Verify-OAuth] Token not found in etcd") + return false, "" + } + + // Get the actual value to check encryption format + command = fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --print-value-only | head -c 20", + etcdKey, + ) + + value, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-OAuth] Failed to get token value from etcd: %v", err) + return false, "" + } + + // Check for KMSv2 encryption prefix + if strings.HasPrefix(value, "k8s:enc:kms:v2:") { + Logf("[Verify-OAuth] ✓ OAuth token is encrypted with KMSv2") + return true, "k8s:enc:kms:v2:" + } else if strings.HasPrefix(value, "k8s:enc:kms:v1:") { + Logf("[Verify-OAuth] OAuth token is encrypted with KMSv1") + return true, "k8s:enc:kms:v1:" + } else if strings.HasPrefix(value, "k8s:enc:") { + Logf("[Verify-OAuth] OAuth token is encrypted with format: %s", value[:15]) + return true, value[:15] + } + + Logf("[Verify-OAuth] OAuth token is not encrypted (no k8s:enc: prefix)") + return false, "" +} diff --git a/test/extended/tests-extension/go.mod b/test/extended/tests-extension/go.mod index 966ae55dd..b576e3812 100644 --- a/test/extended/tests-extension/go.mod +++ b/test/extended/tests-extension/go.mod @@ -3,12 +3,18 @@ module github.com/openshift/cluster-kube-apiserver-operator/test/extended/tests- go 1.24.0 require ( + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.28.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0 + github.com/aws/aws-sdk-go-v2/service/iam v1.38.2 + github.com/aws/aws-sdk-go-v2/service/kms v1.37.2 + github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.26.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 - github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 - github.com/openshift/cluster-kube-apiserver-operator v0.0.0-00010101000000-000000000000 github.com/spf13/cobra v1.9.1 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.34.1 k8s.io/apimachinery v0.34.1 k8s.io/client-go v0.34.1 @@ -17,6 +23,16 @@ require ( require ( cel.dev/expr v0.24.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect @@ -38,13 +54,9 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 // indirect - github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect - github.com/stretchr/testify v1.10.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/test/extended/tests-extension/go.sum b/test/extended/tests-extension/go.sum index e5f650f53..30f8478e5 100644 --- a/test/extended/tests-extension/go.sum +++ b/test/extended/tests-extension/go.sum @@ -2,6 +2,40 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= +github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0 h1:cA4hWo269CN5RY7Arqt8BfzXF0KIN8DSNo/KcqHKkWk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0/go.mod h1:ossaD9Z1ugYb6sq9QIqQLEOorCGcqUoxlhud9M9yE70= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.2 h1:8iFKuRj/FJipy/aDZ2lbq0DYuEHdrxp0qVsdi+ZEwnE= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.2/go.mod h1:UBe4z0VZnbXGp6xaCW1ulE9pndjfpsnrU206rWZcR0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4= +github.com/aws/aws-sdk-go-v2/service/kms v1.37.2 h1:tfBABi5R6aSZlhgTWHxL+opYUDOnIGoNcJLwVYv0jLM= +github.com/aws/aws-sdk-go-v2/service/kms v1.37.2/go.mod h1:dZYFcQwuoh+cLOlFnZItijZptmyDhRIkOKWFO1CfzV8= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.26.2 h1:SW+bplzotcNwVKph3FWsE4Zfk728edeFUCM5VmjbFy0= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.26.2/go.mod h1:cgPfPTC/V3JqwCKed7Q6d0FrgarV7ltz4Bz6S4Q+Dqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,12 +100,6 @@ github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 h1:3athg6KQ+TaNfW4BWZDlGFt1ImSZEJWgzXtPC1VPITI= github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= -github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 h1:Ot2fbEEPmF3WlPQkyEW/bUCV38GMugH/UmZvxpWceNc= -github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= -github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 h1:9JBeIXmnHlpXTQPi7LPmu1jdxznBhAE7bb1K+3D8gxY= -github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235/go.mod h1:L49W6pfrZkfOE5iC1PqEkuLkXG4W0BX4w8b+L2Bv7fM= -github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 h1:bANtDc8SgetSK4nQehf59x3+H9FqVJCprgjs49/OTg0= -github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5/go.mod h1:OlFFws1AO51uzfc48MsStGE4SFMWlMZD0+f5a/zCtKI= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20250416174521-4eb003743b54 h1:ehXndVZfIk/fo18YJCMJ+6b8HL8tzqjP7yWgchMnfCc= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20250416174521-4eb003743b54/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -103,8 +131,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -163,6 +189,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/extended/tests-extension/kms_tests.go b/test/extended/tests-extension/kms_tests.go new file mode 100644 index 000000000..b4c0fc688 --- /dev/null +++ b/test/extended/tests-extension/kms_tests.go @@ -0,0 +1,560 @@ +package extended + +import ( + "context" + "fmt" + "os" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// findExistingKMSKey attempts to find an existing KMS key configured in the APIServer +func findExistingKMSKey(ctx context.Context, node ComputeNode) (string, error) { + // This is a placeholder that would check if KMS is already configured + // In a real implementation, this would check the apiserver CR for existing KMS config + // For now, we'll return empty to indicate no existing key found + return "", fmt.Errorf("no existing KMS key found") +} + +var _ = g.Describe("[Jira:kube-apiserver][sig-api-machinery] API Server KMS", func() { + var ( + kubeClient *kubernetes.Clientset + dynamicClient dynamic.Interface + ctx context.Context + tmpdir string + + // Suite-level shared resources + kmsKeyArn string + kmsRegion string + masterNode ComputeNode + featureGateWasEnabled bool + ) + + // BeforeSuite runs once before all tests in this suite + g.BeforeEach(func() { + if kubeClient != nil { + return // Already initialized + } + + ctx = context.Background() + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE INITIALIZATION ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + // Create temporary directory for test files + var err error + tmpdir, err = os.MkdirTemp("", "kms-test-*") + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Suite-Setup] Created temporary directory: %s", tmpdir) + + // Get kubeconfig and create clients + kubeconfig := GetKubeConfig() + kubeClient, dynamicClient = CreateKubernetesClients(kubeconfig) + + // Check cluster health + g.By("Checking cluster health before KMS test suite") + Logf("[Suite-Setup] Performing cluster health check...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Suite-Setup] Cluster health check failed: %v", err) + g.Skip(fmt.Sprintf("Cluster health check failed: %s", err)) + } + Logf("[Suite-Setup] ✓ Cluster is healthy") + + // Step 1: Check and enable KMS feature gate + Logf("\n--- Step 1: Feature Gate Configuration ---") + g.By("Checking if KMSEncryptionProvider feature gate is enabled") + + isEnabled, err := isFeatureGateEnabled(ctx, dynamicClient, "KMSEncryptionProvider") + o.Expect(err).NotTo(o.HaveOccurred()) + + if isEnabled { + Logf("[Suite-Setup] ✓ KMSEncryptionProvider is already enabled") + featureGateWasEnabled = true + } else { + Logf("[Suite-Setup] KMSEncryptionProvider is not enabled, enabling now...") + featureGateWasEnabled = false + + err = patchFeatureGate(ctx, dynamicClient, `{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":["KMSEncryptionProvider"],"disabled":[]}}}`) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Suite-Setup] ✓ Feature gate patch applied") + + // Wait for kube-apiserver to rollout + g.By("Waiting for kube-apiserver operator to rollout after enabling KMS") + expectedStatus := map[string]string{"Progressing": "True"} + kubeApiserverCoStatus := map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + + Logf("[Suite-Setup] Waiting for operator to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Suite-Setup] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Suite-Setup] Waiting for operator to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Suite-Setup] ✓ kube-apiserver operator is stable after enabling feature gate") + + // Verify all cluster operators are still stable after feature gate change + g.By("Verifying all cluster operators are stable after feature gate change") + Logf("[Suite-Setup] Checking cluster stability after feature gate change...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Suite-Setup] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Suite-Setup] ✓ All cluster operators are stable after feature gate change") + } + + // Step 2: Get master node + Logf("\n--- Step 2: Master Node Discovery ---") + g.By("Getting master nodes") + nodes, cleanup := GetNodes(ctx, kubeClient, dynamicClient, "master") + if cleanup != nil { + g.DeferCleanup(cleanup) + } + o.Expect(len(nodes)).To(o.BeNumerically(">", 0), "No master nodes found") + masterNode = nodes[0] + Logf("[Suite-Setup] ✓ Using master node: %s", masterNode.GetName()) + + // Step 3: Check and create KMS key + Logf("\n--- Step 3: KMS Key Configuration ---") + g.By("Checking if KMS key already exists") + + existingKeyArn, err := findExistingKMSKey(ctx, masterNode) + if err == nil && existingKeyArn != "" { + Logf("[Suite-Setup] ✓ Found existing KMS key: %s", existingKeyArn) + kmsKeyArn = existingKeyArn + } else { + Logf("[Suite-Setup] No existing KMS key found, creating new one...") + kmsKeyArn = masterNode.CreateKMSKey() + o.Expect(kmsKeyArn).NotTo(o.BeEmpty()) + Logf("[Suite-Setup] ✓ Created KMS key: %s", kmsKeyArn) + + Logf("[Suite-Setup] Updating KMS key policy...") + masterNode.UpdateKmsPolicy(kmsKeyArn) + Logf("[Suite-Setup] ✓ KMS key policy updated") + } + + // Extract region from ARN + kmsRegion = masterNode.GetRegionFromARN(kmsKeyArn) + Logf("[Suite-Setup] ✓ KMS region: %s", kmsRegion) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE INITIALIZATION COMPLETE ║") + Logf("║ Feature Gate: %s ║", getStatusString(isEnabled)) + Logf("║ KMS Key ARN: %s", truncateString(kmsKeyArn, 45)) + Logf("║ KMS Region: %-47s║", kmsRegion) + Logf("╚════════════════════════════════════════════════════════════╝\n") + + // Register cleanup to run after all tests complete + g.DeferCleanup(func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE CLEANUP ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + if tmpdir != "" { + os.RemoveAll(tmpdir) + Logf("[Suite-Cleanup] Cleaned up temporary directory: %s", tmpdir) + } + + // Delete KMS key if it was created + if kmsKeyArn != "" { + Logf("[Suite-Cleanup] Deleting KMS key: %s", kmsKeyArn) + masterNode.DeleteKMSKey(kmsKeyArn) + } + + // Only disable feature gate if we enabled it + if !featureGateWasEnabled && dynamicClient != nil { + Logf("[Suite-Cleanup] Disabling KMSEncryptionProvider feature gate...") + err := patchFeatureGate(ctx, dynamicClient, `{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":[],"disabled":["KMSEncryptionProvider"]}}}`) + if err != nil { + Logf("[Suite-Cleanup] Warning: Failed to disable feature gate: %v", err) + } else { + Logf("[Suite-Cleanup] ✓ Feature gate disabled") + + // Wait for rollout + expectedStatus := map[string]string{"Progressing": "True"} + kubeApiserverCoStatus := map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + + Logf("[Suite-Cleanup] Waiting for operator to stabilize (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Suite-Cleanup] Warning: Operator did not start progressing: %v", err) + } + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + if err != nil { + Logf("[Suite-Cleanup] Warning: Operator did not stabilize: %v", err) + } else { + Logf("[Suite-Cleanup] ✓ Operator is stable") + + // Verify all cluster operators are stable after disabling feature gate + Logf("[Suite-Cleanup] Checking cluster stability after disabling feature gate...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Suite-Cleanup] Warning: Cluster stability check failed: %v", err) + } else { + Logf("[Suite-Cleanup] ✓ All cluster operators are stable after cleanup") + } + } + } + } + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE CLEANUP COMPLETE ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) + }) + + g.It("should validate KMS encryption configuration [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:30m]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS ENCRYPTION CONFIGURATION VALIDATION TEST ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + Logf("\n--- Phase 1: Validate KMS Configuration Errors ---") + g.By("Loading KMS test cases from YAML") + Logf("[Phase 1] Loading test cases from YAML file") + + testCases, err := masterNode.LoadKMSTestCasesFromYAML() + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 1] Loaded %d test case(s)", len(testCases)) + + g.By("Running KMS validation test cases") + for i, tc := range testCases { + Logf("\n[Phase 1] Test Case %d/%d: %s", i+1, len(testCases), tc.Name) + g.By(fmt.Sprintf("Testing: %s", tc.Name)) + Logf("[Phase 1] Expected error: %s", tc.ExpectedError) + + // Try to apply the config - should fail with expected error + err = applyAPIServerConfig(ctx, dynamicClient, []byte(tc.Initial)) + if err == nil { + Logf("[Phase 1] ✗ FAILED: Expected validation error but got success") + } else { + Logf("[Phase 1] Actual error: %s", err.Error()) + } + + o.Expect(err).To(o.HaveOccurred(), "Expected validation error for test case: %s", tc.Name) + o.Expect(err.Error()).To(o.ContainSubstring(tc.ExpectedError), + "Error message should contain expected validation error") + + Logf("[Phase 1] ✓ Validation passed") + g.By(fmt.Sprintf("✓ Validation passed for: %s", tc.Name)) + } + Logf("[Phase 1] ✓ All %d validation test cases passed", len(testCases)) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ KMS ENCRYPTION VALIDATION COMPLETED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + g.By("KMS encryption configuration validation completed successfully") + }) + + g.It("should encrypt secrets using KMS [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:60m][Serial][Disruptive]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS SECRET ENCRYPTION VERIFICATION TEST ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + var ( + expectedStatus = map[string]string{"Progressing": "True"} + kubeApiserverCoStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + ) + + Logf("\n--- Phase 1: Apply KMS Encryption Configuration ---") + g.By("Configuring KMS encryption for API server") + Logf("[Phase 1] Using shared KMS key: %s", kmsKeyArn) + Logf("[Phase 1] Using region: %s", kmsRegion) + + // Check and apply KMS config if needed + needsRollout, err := checkAndApplyKMSConfig(ctx, dynamicClient, kmsKeyArn, kmsRegion) + o.Expect(err).NotTo(o.HaveOccurred()) + + if needsRollout { + g.By("Waiting for kube-apiserver operator to rollout") + Logf("[Phase 1] Waiting for kube-apiserver to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Phase 1] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Phase 1] Waiting for kube-apiserver to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not become stable after KMS configuration") + Logf("[Phase 1] ✓ kube-apiserver operator is stable") + + // Check KMS plugin health + Logf("[Phase 1] Checking KMS plugin health...") + isHealthy, healthMsg := checkKMSPluginHealth(ctx, dynamicClient) + if !isHealthy { + Logf("[Phase 1] Warning: KMS plugin health check: %s", healthMsg) + Logf("[Phase 1] Note: KMS socket errors during initial rollout are expected and should resolve") + } else { + Logf("[Phase 1] ✓ KMS plugin health check: %s", healthMsg) + } + } else { + Logf("[Phase 1] ✓ KMS already configured, no rollout needed") + } + + // Verify all cluster operators are still stable after KMS configuration + if needsRollout { + g.By("Verifying all cluster operators are stable after KMS configuration") + Logf("[Phase 1] Checking cluster stability after KMS configuration...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Phase 1] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Phase 1] ✓ All cluster operators are stable after KMS configuration") + } + + Logf("\n--- Phase 2: Verify Encryption Type ---") + g.By("Verifying encryption type is KMS") + encType, encCompleted := masterNode.VerifyEncryptionType(ctx, dynamicClient) + o.Expect(encType).To(o.Equal("KMS"), "Encryption type should be KMS") + Logf("[Phase 2] ✓ Encryption type: %s", encType) + Logf("[Phase 2] ✓ Encryption completed: %v", encCompleted) + + Logf("\n--- Phase 3: Create Test Secret ---") + g.By("Creating test namespace and secret") + testNamespace := "kms-secret-test" + Logf("[Phase 3] Creating namespace: %s", testNamespace) + + err = createNamespace(ctx, kubeClient, testNamespace) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 3] ✓ Namespace created") + + defer func() { + Logf("[Cleanup] Deleting test namespace: %s", testNamespace) + deleteNamespace(ctx, kubeClient, testNamespace) + }() + + secretName := "mysecret1" + secretData := map[string]string{"password": "SuperSecure123"} + Logf("[Phase 3] Creating secret: %s", secretName) + err = createSecret(ctx, kubeClient, testNamespace, secretName, secretData) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 3] ✓ Secret created successfully") + + Logf("\n--- Phase 4: Verify Secret Encryption in etcd ---") + g.By("Verifying secret is encrypted with KMSv2 in etcd") + Logf("[Phase 4] Checking etcd encryption format for secret") + + isEncrypted, encryptionFormat := masterNode.VerifySecretEncryption(ctx, testNamespace, secretName) + o.Expect(isEncrypted).To(o.BeTrue(), "Secret should be encrypted in etcd") + o.Expect(encryptionFormat).To(o.Equal("k8s:enc:kms:v2:"), "Secret should use KMSv2 encryption format") + Logf("[Phase 4] ✓ Secret is encrypted with format: %s", encryptionFormat) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ KMS SECRET ENCRYPTION VERIFIED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) + + g.It("should encrypt OAuthAccessTokens using KMS [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:120m][Serial][Disruptive]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS OAUTH ACCESS TOKEN ENCRYPTION VERIFICATION ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + var ( + expectedStatus = map[string]string{"Progressing": "True"} + kubeApiserverCoStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + userUID string + ) + + Logf("\n--- Phase 1: Ensure KMS Encryption Configuration ---") + g.By("Configuring KMS encryption for API server") + Logf("[Phase 1] Using shared KMS key: %s", kmsKeyArn) + Logf("[Phase 1] Using region: %s", kmsRegion) + + // Check and apply KMS config if needed + needsRollout, err := checkAndApplyKMSConfig(ctx, dynamicClient, kmsKeyArn, kmsRegion) + o.Expect(err).NotTo(o.HaveOccurred()) + + if needsRollout { + g.By("Waiting for kube-apiserver operator to rollout") + Logf("[Phase 1] Waiting for kube-apiserver to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Phase 1] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Phase 1] Waiting for kube-apiserver to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not become stable after KMS configuration") + Logf("[Phase 1] ✓ kube-apiserver operator is stable") + + // Check KMS plugin health + Logf("[Phase 1] Checking KMS plugin health...") + isHealthy, healthMsg := checkKMSPluginHealth(ctx, dynamicClient) + if !isHealthy { + Logf("[Phase 1] Warning: KMS plugin health check: %s", healthMsg) + Logf("[Phase 1] Note: KMS socket errors during initial rollout are expected and should resolve") + } else { + Logf("[Phase 1] ✓ KMS plugin health check: %s", healthMsg) + } + + // Verify all cluster operators are still stable after KMS configuration + g.By("Verifying all cluster operators are stable after KMS configuration") + Logf("[Phase 1] Checking cluster stability after KMS configuration...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Phase 1] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Phase 1] ✓ All cluster operators are stable after KMS configuration") + } else { + Logf("[Phase 1] ✓ KMS already configured, no rollout needed") + } + + Logf("\n--- Phase 2: Generate Token and Resource Name ---") + g.By("Generating secure token and resource name") + + tokenValue, tokenResourceName := generateSecureToken() + Logf("[Phase 2] ✓ Generated token value: %s", maskToken(tokenValue)) + Logf("[Phase 2] ✓ Token resource name: %s", tokenResourceName) + + Logf("\n--- Phase 3: Get Test User Information ---") + g.By("Getting test user UID") + testUser := "test-user-01" + userUID, err = getUserUID(ctx, dynamicClient, testUser) + if err != nil { + Logf("[Phase 3] Test user not found, creating user") + userUID = createTestUser(ctx, dynamicClient, testUser) + } + o.Expect(userUID).NotTo(o.BeEmpty()) + Logf("[Phase 3] ✓ Test user UID: %s", userUID) + + Logf("\n--- Phase 4: Create OAuthAccessToken ---") + g.By("Creating OAuthAccessToken") + + accessToken := createOAuthAccessToken(tokenResourceName, tokenValue, testUser, userUID) + Logf("[Phase 4] Creating access token: %s", tokenResourceName) + + err = applyOAuthToken(ctx, dynamicClient, "oauthaccesstokens", accessToken) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 4] ✓ OAuthAccessToken created successfully") + + defer func() { + Logf("[Cleanup] Deleting OAuthAccessToken: %s", tokenResourceName) + deleteOAuthToken(ctx, dynamicClient, "oauthaccesstokens", tokenResourceName) + }() + + Logf("\n--- Phase 5: Verify Token Encryption in etcd ---") + g.By("Verifying OAuthAccessToken is encrypted with KMSv2 in etcd") + + isEncrypted, encryptionFormat := masterNode.VerifyOAuthTokenEncryption(ctx, "oauthaccesstokens", tokenResourceName) + o.Expect(isEncrypted).To(o.BeTrue(), "OAuthAccessToken should be encrypted in etcd") + o.Expect(encryptionFormat).To(o.Equal("k8s:enc:kms:v2:"), "OAuthAccessToken should use KMSv2 encryption format") + Logf("[Phase 5] ✓ OAuthAccessToken is encrypted with format: %s", encryptionFormat) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ OAUTH ACCESS TOKEN ENCRYPTION VERIFIED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) + + g.It("should encrypt OAuthAuthorizeTokens using KMS [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:120m][Serial][Disruptive]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS OAUTH AUTHORIZE TOKEN ENCRYPTION VERIFICATION ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + var ( + expectedStatus = map[string]string{"Progressing": "True"} + kubeApiserverCoStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + userUID string + ) + + Logf("\n--- Phase 1: Ensure KMS Encryption Configuration ---") + g.By("Configuring KMS encryption for API server") + Logf("[Phase 1] Using shared KMS key: %s", kmsKeyArn) + Logf("[Phase 1] Using region: %s", kmsRegion) + + // Check and apply KMS config if needed + needsRollout, err := checkAndApplyKMSConfig(ctx, dynamicClient, kmsKeyArn, kmsRegion) + o.Expect(err).NotTo(o.HaveOccurred()) + + if needsRollout { + g.By("Waiting for kube-apiserver operator to rollout") + Logf("[Phase 1] Waiting for kube-apiserver to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Phase 1] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Phase 1] Waiting for kube-apiserver to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not become stable after KMS configuration") + Logf("[Phase 1] ✓ kube-apiserver operator is stable") + + // Check KMS plugin health + Logf("[Phase 1] Checking KMS plugin health...") + isHealthy, healthMsg := checkKMSPluginHealth(ctx, dynamicClient) + if !isHealthy { + Logf("[Phase 1] Warning: KMS plugin health check: %s", healthMsg) + Logf("[Phase 1] Note: KMS socket errors during initial rollout are expected and should resolve") + } else { + Logf("[Phase 1] ✓ KMS plugin health check: %s", healthMsg) + } + + // Verify all cluster operators are still stable after KMS configuration + g.By("Verifying all cluster operators are stable after KMS configuration") + Logf("[Phase 1] Checking cluster stability after KMS configuration...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Phase 1] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Phase 1] ✓ All cluster operators are stable after KMS configuration") + } else { + Logf("[Phase 1] ✓ KMS already configured, no rollout needed") + } + + Logf("\n--- Phase 2: Generate Auth Code and Resource Name ---") + g.By("Generating secure authorization code and resource name") + + authCode, authResourceName := generateSecureToken() + Logf("[Phase 2] ✓ Generated auth code: %s", maskToken(authCode)) + Logf("[Phase 2] ✓ Auth token resource name: %s", authResourceName) + + Logf("\n--- Phase 3: Get Test User Information ---") + g.By("Getting test user UID") + testUser := "test-user-01" + userUID, err = getUserUID(ctx, dynamicClient, testUser) + if err != nil { + Logf("[Phase 3] Test user not found, creating user") + userUID = createTestUser(ctx, dynamicClient, testUser) + } + o.Expect(userUID).NotTo(o.BeEmpty()) + Logf("[Phase 3] ✓ Test user UID: %s", userUID) + + Logf("\n--- Phase 4: Create OAuthAuthorizeToken ---") + g.By("Creating OAuthAuthorizeToken") + + authorizeToken := createOAuthAuthorizeToken(authResourceName, authCode, testUser, userUID) + Logf("[Phase 4] Creating authorize token: %s", authResourceName) + + err = applyOAuthToken(ctx, dynamicClient, "oauthauthorizetokens", authorizeToken) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 4] ✓ OAuthAuthorizeToken created successfully") + + defer func() { + Logf("[Cleanup] Deleting OAuthAuthorizeToken: %s", authResourceName) + deleteOAuthToken(ctx, dynamicClient, "oauthauthorizetokens", authResourceName) + }() + + Logf("\n--- Phase 5: Verify Token Encryption in etcd ---") + g.By("Verifying OAuthAuthorizeToken is encrypted with KMSv2 in etcd") + + isEncrypted, encryptionFormat := masterNode.VerifyOAuthTokenEncryption(ctx, "oauthauthorizetokens", authResourceName) + o.Expect(isEncrypted).To(o.BeTrue(), "OAuthAuthorizeToken should be encrypted in etcd") + o.Expect(encryptionFormat).To(o.Equal("k8s:enc:kms:v2:"), "OAuthAuthorizeToken should use KMSv2 encryption format") + Logf("[Phase 5] ✓ OAuthAuthorizeToken is encrypted with format: %s", encryptionFormat) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ OAUTH AUTHORIZE TOKEN ENCRYPTION VERIFIED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) +}) diff --git a/test/extended/tests-extension/testdata/kms_tests_aws.yaml b/test/extended/tests-extension/testdata/kms_tests_aws.yaml new file mode 100644 index 000000000..d1aced61c --- /dev/null +++ b/test/extended/tests-extension/testdata/kms_tests_aws.yaml @@ -0,0 +1,119 @@ +- name: Should fail to create encrypt with KMS for AWS without region + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + expectedError: "spec.encryption.kms.aws.region: Required value" + +- name: Should not allow kms config with encrypt aescbc + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: aescbc + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + region: us-east-1 + expectedError: "kms config is required when encryption type is KMS, and forbidden otherwise" + +- name: Should fail to create with an empty KMS config + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: {} + expectedError: "spec.encryption.kms.type: Required value" + +- name: Should fail to create with kms type AWS but without aws config + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + expectedError: "aws config is required when kms provider type is AWS, and forbidden otherwise" + +- name: Should fail to create AWS KMS without a keyARN + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + region: us-east-1 + expectedError: "spec.encryption.kms.aws.keyARN: Required value" + +- name: Should fail to create AWS KMS with invalid keyARN format + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: not-a-kms-arn + region: us-east-1 + expectedError: "keyARN must follow the format `arn:aws:kms:::key/`. The account ID must be a 12 digit number and the region and key ID should consist only of lowercase hexadecimal characters and hyphens (-)." + +- name: Should fail to create AWS KMS with empty region + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + region: "" + expectedError: "spec.encryption.kms.aws.region in body should be at least 1 chars long" + +- name: Should fail to create AWS KMS with invalid region format + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + region: "INVALID-REGION" + expectedError: "region must be a valid AWS region, consisting of lowercase characters, digits and hyphens (-) only." + diff --git a/test/extended/tests-extension/util.go b/test/extended/tests-extension/util.go new file mode 100644 index 000000000..0033edc1f --- /dev/null +++ b/test/extended/tests-extension/util.go @@ -0,0 +1,802 @@ +package extended + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// Logf logs formatted output to Ginkgo writer +func Logf(format string, args ...interface{}) { + fmt.Fprintf(g.GinkgoWriter, format+"\n", args...) +} + +// Failf fails the test with formatted message +func Failf(format string, args ...interface{}) { + g.Fail(fmt.Sprintf(format, args...)) +} + +// GetKubeConfig gets KUBECONFIG from environment and validates it exists +func GetKubeConfig() string { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + Logf("[Setup] KUBECONFIG not set, skipping test") + g.Skip("KUBECONFIG environment variable not set") + } + Logf("[Setup] Using KUBECONFIG: %s", kubeconfig) + return kubeconfig +} + +// CreateKubernetesClients creates Kubernetes and dynamic clients from kubeconfig +func CreateKubernetesClients(kubeconfig string) (*kubernetes.Clientset, dynamic.Interface) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + Failf("Failed to load Kubernetes config: %v", err) + } + Logf("[Setup] Kubernetes config loaded successfully") + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + Failf("Failed to create Kubernetes client: %v", err) + } + Logf("[Setup] Kubernetes client created") + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + Failf("Failed to create dynamic client: %v", err) + } + Logf("[Setup] Dynamic client created") + + return kubeClient, dynamicClient +} + +// patchFeatureGate patches the cluster featuregate +func patchFeatureGate(ctx context.Context, client dynamic.Interface, patchData string) error { + featureGateGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "featuregates", + } + + _, err := client.Resource(featureGateGVR).Patch(ctx, "cluster", "application/merge-patch+json", + []byte(patchData), v1.PatchOptions{}) + + return err +} + +// applyAPIServerConfig attempts to apply an APIServer configuration +func applyAPIServerConfig(ctx context.Context, client dynamic.Interface, yamlData []byte) error { + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + // Parse YAML to get the desired spec + var yamlObj interface{} + err := yaml.Unmarshal(yamlData, &yamlObj) + if err != nil { + return fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + // Convert to unstructured + yamlMap, err := convertToStringMap(yamlObj) + if err != nil { + return fmt.Errorf("failed to convert YAML to unstructured: %w", err) + } + + // Get the existing APIServer resource to preserve metadata including resourceVersion + existing, err := client.Resource(apiServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get existing apiserver: %w", err) + } + + // Extract spec from YAML and set it on the existing resource + if spec, found := yamlMap["spec"]; found { + existing.Object["spec"] = spec + } + + // Try to update the APIServer resource - this will trigger server-side validation + _, err = client.Resource(apiServerGVR).Update(ctx, existing, v1.UpdateOptions{}) + return err +} + +// convertToStringMap converts interface{} to map[string]interface{} recursively +func convertToStringMap(i interface{}) (map[string]interface{}, error) { + switch x := i.(type) { + case map[interface{}]interface{}: + m := map[string]interface{}{} + for k, v := range x { + strKey, ok := k.(string) + if !ok { + return nil, fmt.Errorf("non-string key found: %v", k) + } + switch val := v.(type) { + case map[interface{}]interface{}: + converted, err := convertToStringMap(val) + if err != nil { + return nil, err + } + m[strKey] = converted + case []interface{}: + m[strKey] = convertSlice(val) + default: + m[strKey] = val + } + } + return m, nil + case map[string]interface{}: + return x, nil + default: + return nil, fmt.Errorf("expected map, got %T", i) + } +} + +// convertSlice converts []interface{} recursively +func convertSlice(s []interface{}) []interface{} { + result := make([]interface{}, len(s)) + for i, v := range s { + switch val := v.(type) { + case map[interface{}]interface{}: + converted, _ := convertToStringMap(val) + result[i] = converted + case []interface{}: + result[i] = convertSlice(val) + default: + result[i] = val + } + } + return result +} + +// waitForClusterStable waits for the cluster to be stable +// Checks that all cluster operators are Available=True, Progressing=False, Degraded=False +func waitForClusterStable(ctx context.Context, client dynamic.Interface) error { + coGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "clusteroperators", + } + + // Wait for all COs to be stable + return wait.PollUntilContextTimeout(ctx, 10*time.Second, 18*time.Minute, true, + func(ctx context.Context) (bool, error) { + coList, err := client.Resource(coGVR).List(ctx, v1.ListOptions{}) + if err != nil { + return false, nil + } + + unstableCOs := []string{} + for _, item := range coList.Items { + coName := item.GetName() + conditions, found, err := unstructured.NestedSlice(item.Object, "status", "conditions") + if !found || err != nil { + unstableCOs = append(unstableCOs, coName) + continue + } + + currentStatus := make(map[string]string) + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + condType := condition["type"].(string) + status := condition["status"].(string) + currentStatus[condType] = status + } + + // Check if CO is stable (Available=True, Progressing=False, Degraded=False) + if currentStatus["Available"] != "True" || + currentStatus["Progressing"] != "False" || + currentStatus["Degraded"] != "False" { + unstableCOs = append(unstableCOs, coName) + } + } + + if len(unstableCOs) > 0 { + return false, nil + } + + return true, nil + }) +} + +// waitForOperatorStatus waits for an operator to reach the expected status +func waitForOperatorStatus(ctx context.Context, client dynamic.Interface, operatorName string, timeoutSeconds int, expectedStatus map[string]string) error { + coGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "clusteroperators", + } + + return wait.PollUntilContextTimeout(ctx, 10*time.Second, time.Duration(timeoutSeconds)*time.Second, true, + func(ctx context.Context) (bool, error) { + co, err := client.Resource(coGVR).Get(ctx, operatorName, v1.GetOptions{}) + if err != nil { + return false, nil + } + + conditions, found, err := unstructured.NestedSlice(co.Object, "status", "conditions") + if !found || err != nil { + return false, nil + } + + currentStatus := make(map[string]string) + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + condType := condition["type"].(string) + status := condition["status"].(string) + currentStatus[condType] = status + } + + // Check if current status matches expected status + for expectedType, expectedValue := range expectedStatus { + if currentStatus[expectedType] != expectedValue { + return false, nil + } + } + + return true, nil + }) +} + +// checkAndApplyKMSConfig checks if KMS config is already applied and correct, applies if needed +// Returns: (needsRollout bool, error) +func checkAndApplyKMSConfig(ctx context.Context, client dynamic.Interface, expectedKeyARN, expectedRegion string) (bool, error) { + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + // Get current apiserver config + apiServer, err := client.Resource(apiServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get apiserver config: %w", err) + } + + // Check current encryption config + encType, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "type") + if encType == "KMS" { + // KMS is already configured, verify it's correct + kmsType, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "kms", "type") + if kmsType == "AWS" { + currentKeyARN, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "kms", "aws", "keyARN") + currentRegion, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "kms", "aws", "region") + + if currentKeyARN == expectedKeyARN && currentRegion == expectedRegion { + Logf("[KMS-Config] ✓ KMS is already configured correctly") + Logf("[KMS-Config] Key ARN: %s", currentKeyARN) + Logf("[KMS-Config] Region: %s", currentRegion) + return false, nil // No need to apply, already correct + } + + Logf("[KMS-Config] KMS is configured but with different values") + Logf("[KMS-Config] Current Key ARN: %s", currentKeyARN) + Logf("[KMS-Config] Expected Key ARN: %s", expectedKeyARN) + Logf("[KMS-Config] Current Region: %s", currentRegion) + Logf("[KMS-Config] Expected Region: %s", expectedRegion) + } + } else if encType != "" { + Logf("[KMS-Config] Current encryption type: %s (not KMS)", encType) + } else { + Logf("[KMS-Config] No encryption currently configured") + } + + // Apply KMS configuration + Logf("[KMS-Config] Applying KMS encryption configuration...") + kmsConfig := fmt.Sprintf(`{ + "spec": { + "encryption": { + "type": "KMS", + "kms": { + "type": "AWS", + "aws": { + "keyARN": "%s", + "region": "%s" + } + } + } + } + }`, expectedKeyARN, expectedRegion) + + err = patchAPIServerConfig(ctx, client, kmsConfig) + if err != nil { + return false, fmt.Errorf("failed to apply KMS config: %w", err) + } + + Logf("[KMS-Config] ✓ KMS configuration applied") + return true, nil // Config was applied, rollout needed +} + +// patchAPIServerConfig patches the API server configuration +func patchAPIServerConfig(ctx context.Context, client dynamic.Interface, patchData string) error { + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + _, err := client.Resource(apiServerGVR).Patch(ctx, "cluster", "application/merge-patch+json", + []byte(patchData), v1.PatchOptions{}) + + return err +} + +// createNamespace creates a namespace +func createNamespace(ctx context.Context, kubeClient *kubernetes.Clientset, namespace string) error { + ns := &corev1.Namespace{ + ObjectMeta: v1.ObjectMeta{ + Name: namespace, + }, + } + _, err := kubeClient.CoreV1().Namespaces().Create(ctx, ns, v1.CreateOptions{}) + return err +} + +// deleteNamespace deletes a namespace +func deleteNamespace(ctx context.Context, kubeClient *kubernetes.Clientset, namespace string) error { + return kubeClient.CoreV1().Namespaces().Delete(ctx, namespace, v1.DeleteOptions{}) +} + +// createSecret creates a secret in the specified namespace +func createSecret(ctx context.Context, kubeClient *kubernetes.Clientset, namespace, name string, data map[string]string) error { + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + StringData: data, + Type: corev1.SecretTypeOpaque, + } + _, err := kubeClient.CoreV1().Secrets(namespace).Create(ctx, secret, v1.CreateOptions{}) + return err +} + +// generateSecureToken generates a secure token and its resource name +// Returns: (tokenValue, tokenResourceName) +func generateSecureToken() (string, string) { + // Generate 32-byte random token + rawToken := make([]byte, 32) + _, err := rand.Read(rawToken) + if err != nil { + Failf("Failed to generate random token: %v", err) + } + + // Base64-URL encode the token + tokenValue := base64.URLEncoding.EncodeToString(rawToken) + tokenValue = strings.TrimRight(tokenValue, "=") + + // Calculate SHA256 hash of token + hash := sha256.Sum256([]byte(tokenValue)) + + // Base64-URL encode the hash + tokenHash := base64.URLEncoding.EncodeToString(hash[:]) + tokenHash = strings.TrimRight(tokenHash, "=") + + // Create resource name with sha256~ prefix + resourceName := "sha256~" + tokenHash + + return tokenValue, resourceName +} + +// maskToken masks a token for logging (shows first and last 4 characters) +func maskToken(token string) string { + if len(token) <= 8 { + return "****" + } + return token[:4] + "..." + token[len(token)-4:] +} + +// getUserUID gets the UID of a user +func getUserUID(ctx context.Context, client dynamic.Interface, userName string) (string, error) { + userGVR := schema.GroupVersionResource{ + Group: "user.openshift.io", + Version: "v1", + Resource: "users", + } + + user, err := client.Resource(userGVR).Get(ctx, userName, v1.GetOptions{}) + if err != nil { + return "", err + } + + uid, found, err := unstructured.NestedString(user.Object, "metadata", "uid") + if !found || err != nil { + return "", fmt.Errorf("uid not found in user object") + } + + return uid, nil +} + +// createTestUser creates a test user (simplified - in real cluster this would be done via IDP) +func createTestUser(ctx context.Context, client dynamic.Interface, userName string) string { + // In a real cluster, users come from IDP + // For testing, we'll use a well-known test user UID + // This is a placeholder - actual implementation depends on cluster setup + Logf("Using placeholder UID for test user: %s", userName) + return "00000000-0000-0000-0000-000000000001" +} + +// createOAuthAccessToken creates an OAuthAccessToken object +func createOAuthAccessToken(resourceName, tokenValue, userName, userUID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "oauth.openshift.io/v1", + "kind": "OAuthAccessToken", + "metadata": map[string]interface{}{ + "name": resourceName, + }, + "clientName": "openshift-challenging-client", + "userName": userName, + "userUID": userUID, + "scopes": []interface{}{ + "user:full", + }, + "expiresIn": 86400, // 24 hours + "redirectURI": "https://oauth-openshift.apps.example.com/oauth/token/implicit", + "token": tokenValue, + }, + } +} + +// createOAuthAuthorizeToken creates an OAuthAuthorizeToken object +func createOAuthAuthorizeToken(resourceName, authCode, userName, userUID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "oauth.openshift.io/v1", + "kind": "OAuthAuthorizeToken", + "metadata": map[string]interface{}{ + "name": resourceName, + }, + "clientName": "openshift-challenging-client", + "userName": userName, + "userUID": userUID, + "scopes": []interface{}{ + "user:full", + }, + "expiresIn": 300, // 5 minutes + "redirectURI": "https://oauth-openshift.apps.example.com/oauth/token/implicit", + "code": authCode, + }, + } +} + +// applyOAuthToken applies an OAuth token (access or authorize) +func applyOAuthToken(ctx context.Context, client dynamic.Interface, resource string, token *unstructured.Unstructured) error { + oauthGVR := schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: resource, + } + + _, err := client.Resource(oauthGVR).Create(ctx, token, v1.CreateOptions{}) + return err +} + +// deleteOAuthToken deletes an OAuth token +func deleteOAuthToken(ctx context.Context, client dynamic.Interface, resource, name string) error { + oauthGVR := schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: resource, + } + + return client.Resource(oauthGVR).Delete(ctx, name, v1.DeleteOptions{}) +} + +// checkKMSPluginHealth checks if KMS plugin pods are healthy in kube-apiserver +// Returns: (isHealthy, message) +func checkKMSPluginHealth(ctx context.Context, client dynamic.Interface) (bool, string) { + kubeAPIServerGVR := schema.GroupVersionResource{ + Group: "operator.openshift.io", + Version: "v1", + Resource: "kubeapiservers", + } + + kubeAPIServer, err := client.Resource(kubeAPIServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return false, fmt.Sprintf("Failed to get kubeapiserver: %v", err) + } + + // Check conditions for KMS health + conditions, found, err := unstructured.NestedSlice(kubeAPIServer.Object, "status", "conditions") + if !found || err != nil { + return false, "KMS conditions not found in kubeapiserver status" + } + + // Look for KMS-related error conditions + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + condType, _ := condition["type"].(string) + status, _ := condition["status"].(string) + message, _ := condition["message"].(string) + reason, _ := condition["reason"].(string) + + // Check for Degraded condition related to KMS + if condType == "Degraded" && status == "True" { + if strings.Contains(message, "kms-provider") || strings.Contains(message, "kms") { + return false, fmt.Sprintf("KMS degraded: %s - %s", reason, message) + } + } + + // Check for KMSConnectionDegraded or similar conditions + if strings.Contains(condType, "KMS") && status == "True" { + return false, fmt.Sprintf("KMS condition %s: %s - %s", condType, reason, message) + } + } + + return true, "KMS plugin appears healthy" +} + +// verifyEncryptionType verifies the encryption type configured in the APIServer +// This is a generic function that works across all cloud platforms +// Returns: (encryptionType, encryptionCompleted) +func verifyEncryptionType(ctx context.Context, client dynamic.Interface) (string, bool) { + Logf("[Verify] Checking encryption configuration") + + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + apiServer, err := client.Resource(apiServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + Logf("[Verify] Failed to get apiserver: %v", err) + return "", false + } + + // Get encryption type + encType, found, err := unstructured.NestedString(apiServer.Object, "spec", "encryption", "type") + if !found || err != nil { + Logf("[Verify] Encryption type not found in spec") + return "", false + } + + // Check kubeapiserver for encryption status + kubeAPIServerGVR := schema.GroupVersionResource{ + Group: "operator.openshift.io", + Version: "v1", + Resource: "kubeapiservers", + } + + kubeAPIServer, err := client.Resource(kubeAPIServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + Logf("[Verify] Failed to get kubeapiserver: %v", err) + return encType, false + } + + // Get encryption conditions + conditions, found, err := unstructured.NestedSlice(kubeAPIServer.Object, "status", "conditions") + if !found || err != nil { + Logf("[Verify] Conditions not found in kubeapiserver status") + return encType, false + } + + // Check for Encrypted condition + encryptionCompleted := false + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + if condition["type"] == "Encrypted" { + reason, _ := condition["reason"].(string) + if reason == "EncryptionCompleted" { + encryptionCompleted = true + message, _ := condition["message"].(string) + Logf("[Verify] Encryption status: %s - %s", reason, message) + } + } + } + + return encType, encryptionCompleted +} + +// isFeatureGateEnabled checks if a specific feature gate is enabled +func isFeatureGateEnabled(ctx context.Context, client dynamic.Interface, featureName string) (bool, error) { + featureGateGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "featuregates", + } + + featureGate, err := client.Resource(featureGateGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get feature gate: %w", err) + } + + // Check status for enabled features + featureGates, found, err := unstructured.NestedSlice(featureGate.Object, "status", "featureGates") + if !found || err != nil { + return false, nil + } + + for _, fg := range featureGates { + fgDetails := fg.(map[string]interface{}) + enabled, found, err := unstructured.NestedSlice(fgDetails, "enabled") + if !found || err != nil { + continue + } + + for _, feature := range enabled { + featureMap := feature.(map[string]interface{}) + if name, found := featureMap["name"]; found && name == featureName { + return true, nil + } + } + } + + return false, nil +} + +// getStatusString returns a formatted status string +func getStatusString(enabled bool) string { + if enabled { + return "Already Enabled" + } + return "Newly Enabled " +} + +// truncateString truncates a string to the specified length +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + // Pad with spaces to maintain box alignment + return fmt.Sprintf("%-"+fmt.Sprint(maxLen)+"s║", s) + } + return s[:maxLen-3] + "...║" +} + +// debugNode executes a command on a node using oc debug with chroot +// Returns stdout, stderr, and error +func debugNode(nodeName string, cmd ...string) (string, string, error) { + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Build the oc debug command arguments + // oc debug node/ -- chroot /host + args := []string{"debug", fmt.Sprintf("node/%s", nodeName), "--", "chroot", "/host"} + args = append(args, cmd...) + + Logf("[DebugNode] Executing: oc %s", strings.Join(args, " ")) + + command := exec.CommandContext(ctx, "oc", args...) + + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + + err := command.Run() + + return stdout.String(), stderr.String(), err +} + +// debugNodeRetryWithChroot executes a command on a node with retry logic +// Similar to compat_otp.DebugNodeRetryWithOptionsAndChroot +func debugNodeRetryWithChroot(nodeName string, cmd ...string) (string, error) { + var stdErr string + var stdOut string + var err error + + // Retry logic with polling + errWait := wait.Poll(3*time.Second, 30*time.Second, func() (bool, error) { + stdOut, stdErr, err = debugNode(nodeName, cmd...) + if err != nil { + Logf("[DebugNode] Retry attempt failed: %v", err) + return false, nil // Retry + } + return true, nil // Success + }) + + if errWait != nil { + return "", fmt.Errorf("failed to debug node after retries: %w", errWait) + } + + // Combine stdout and stderr + return strings.Join([]string{stdOut, stdErr}, "\n"), err +} + +// executeNodeCommand executes a command on a node using oc debug with chroot +func executeNodeCommand(nodeName, command string) (string, error) { + Logf("[Exec] Running command on node %s", nodeName) + Logf("[Exec] Command: %s", command) + + // Execute the command with retry + output, err := debugNodeRetryWithChroot(nodeName, "/bin/bash", "-c", command) + if err != nil { + Logf("[Exec] Command failed: %v", err) + return output, fmt.Errorf("failed to execute command on node %s: %w", nodeName, err) + } + + Logf("[Exec] Command completed successfully") + return output, nil +} + +// getAwsCredentialFromCluster retrieves AWS credentials from the cluster's kube-system namespace +// and sets them as environment variables +func getAwsCredentialFromCluster() error { + Logf("[AWS-Creds] Retrieving AWS credentials from cluster") + + // Get the aws-creds secret from kube-system namespace + cmd := exec.Command("oc", "get", "secret/aws-creds", "-n", "kube-system", "-o", "json") + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + // Skip for STS and C2S clusters + Logf("[AWS-Creds] Did not get credential to access AWS: %v", err) + g.Skip("Did not get credential to access AWS, skip the testing.") + return fmt.Errorf("failed to get AWS credentials from cluster: %w", err) + } + + // Parse the JSON output + var secret map[string]interface{} + if err := json.Unmarshal(stdout.Bytes(), &secret); err != nil { + return fmt.Errorf("failed to parse secret JSON: %w", err) + } + + // Extract base64-encoded credentials + data, ok := secret["data"].(map[string]interface{}) + if !ok { + return fmt.Errorf("secret data not found") + } + + accessKeyIDBase64, ok1 := data["aws_access_key_id"].(string) + secureKeyBase64, ok2 := data["aws_secret_access_key"].(string) + if !ok1 || !ok2 { + return fmt.Errorf("AWS credentials not found in secret") + } + + // Decode base64 credentials + accessKeyID, err := base64.StdEncoding.DecodeString(accessKeyIDBase64) + if err != nil { + return fmt.Errorf("failed to decode access key ID: %w", err) + } + + secureKey, err := base64.StdEncoding.DecodeString(secureKeyBase64) + if err != nil { + return fmt.Errorf("failed to decode secret access key: %w", err) + } + + // Get AWS region from infrastructure resource + cmd = exec.Command("oc", "get", "infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}") + stdout.Reset() + stderr.Reset() + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + if err != nil { + return fmt.Errorf("failed to get AWS region: %w", err) + } + + clusterRegion := strings.TrimSpace(stdout.String()) + + // Set environment variables + os.Setenv("AWS_ACCESS_KEY_ID", string(accessKeyID)) + os.Setenv("AWS_SECRET_ACCESS_KEY", string(secureKey)) + os.Setenv("AWS_REGION", clusterRegion) + + Logf("[AWS-Creds] ✓ AWS credentials set successfully") + Logf("[AWS-Creds] Region: %s", clusterRegion) + + return nil +}