🌱 update e2e for gc (#494)

* update e2e for gc

Signed-off-by: Zhiwei Yin <[email protected]>

* wait clustermanager is applied

Signed-off-by: Zhiwei Yin <[email protected]>

---------

Signed-off-by: Zhiwei Yin <[email protected]>

Author: zhiweiyin318

go.mod

@@ -16,23 +16,23 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/pflag v1.0.5
-	google.golang.org/grpc v1.67.0
+	google.golang.org/grpc v1.68.1
 	gopkg.in/yaml.v2 v2.4.0
 	helm.sh/helm/v3 v3.17.3
-	k8s.io/api v0.32.3
+	k8s.io/api v0.32.4
 	k8s.io/apiextensions-apiserver v0.32.2
-	k8s.io/apimachinery v0.32.3
+	k8s.io/apimachinery v0.32.4
 	k8s.io/cli-runtime v0.32.2
-	k8s.io/client-go v0.32.3
-	k8s.io/component-base v0.32.3
+	k8s.io/client-go v0.32.4
+	k8s.io/component-base v0.32.4
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubectl v0.32.2
 	k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738
-	open-cluster-management.io/api v0.16.2-0.20250422072120-cadf714c3055
+	open-cluster-management.io/api v1.0.0
 	open-cluster-management.io/cluster-proxy v0.7.0
 	open-cluster-management.io/managed-serviceaccount v0.8.0
-	open-cluster-management.io/ocm v0.16.1-0.20250422150056-f4b6dcb15929
-	open-cluster-management.io/sdk-go v0.16.1-0.20250411154302-3a424961ead4
+	open-cluster-management.io/ocm v1.0.0
+	open-cluster-management.io/sdk-go v1.0.0
 	sigs.k8s.io/apiserver-network-proxy v0.29.0
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0
 	sigs.k8s.io/controller-runtime v0.20.2
@@ -87,13 +87,13 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gorilla/mux v1.8.1 // indirect
-	github.com/gorilla/websocket v1.5.1 // indirect
+	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -141,32 +141,34 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/xlab/treeprint v1.2.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
-	go.opentelemetry.io/otel v1.28.0 // indirect
-	go.opentelemetry.io/otel/metric v1.28.0 // indirect
-	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
+	go.opentelemetry.io/otel v1.33.0 // indirect
+	go.opentelemetry.io/otel/metric v1.33.0 // indirect
+	go.opentelemetry.io/otel/trace v1.33.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
-	golang.org/x/crypto v0.36.0 // indirect
-	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/crypto v0.37.0 // indirect
+	golang.org/x/net v0.38.0 // indirect
 	golang.org/x/oauth2 v0.28.0 // indirect
-	golang.org/x/sync v0.12.0 // indirect
+	golang.org/x/sync v0.13.0 // indirect
 	golang.org/x/sys v0.32.0 // indirect
-	golang.org/x/term v0.30.0 // indirect
-	golang.org/x/text v0.23.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
+	golang.org/x/term v0.31.0 // indirect
+	golang.org/x/text v0.24.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
 	golang.org/x/tools v0.31.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.32.3 // indirect
 	k8s.io/kube-aggregator v0.32.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect
 	sigs.k8s.io/kustomize/api v0.18.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -4,6 +4,7 @@ package clusteradme2e
 import (
 	"context"
 	"fmt"
+	"open-cluster-management.io/clusteradm/test/e2e/util"
 	"time"
 
 	"github.com/onsi/ginkgo/v2"
@@ -35,6 +36,8 @@ var _ = ginkgo.Describe("test clusteradm with addon create", func() {
 			)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
+			util.WaitClusterManagerApplied(operatorClient)
+
 			ginkgo.By("managedcluster1 join hub")
 			err = e2e.Clusteradm().Join(
 				"--context", e2e.Cluster().ManagedCluster1().Context(),

test/e2e/clusteradm/addon_create_test.go

@@ -32,7 +32,11 @@ var _ = ginkgo.Describe("test clusteradm with bootstrap token in singleton mode"
 
 			cm, err := operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-			gomega.Expect(len(cm.Spec.RegistrationConfiguration.FeatureGates)).Should(gomega.Equal(1))
+
+			util.WaitClusterManagerApplied(operatorClient)
+
+			// 2 featureGates: DefaultClusterSet and ResourceCleanup
+			gomega.Expect(len(cm.Spec.RegistrationConfiguration.FeatureGates)).Should(gomega.Equal(2))
 
 			err = e2e.Clusteradm().Init(
 				"--use-bootstrap-token",
@@ -113,7 +117,8 @@ var _ = ginkgo.Describe("test clusteradm with bootstrap token in singleton mode"
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 			cm, err = operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
 			gomega.Expect(err).NotTo(gomega.HaveOccurred())
-			gomega.Expect(len(cm.Spec.RegistrationConfiguration.FeatureGates)).Should(gomega.Equal(2))
+			// 3 featureGates: ManagedClusterAutoApproval, DefaultClusterSet and ResourceCleanup
+			gomega.Expect(len(cm.Spec.RegistrationConfiguration.FeatureGates)).Should(gomega.Equal(3))
 
 			gomega.Eventually(func() error {
 				return util.ValidateImagePullSecret(kubeClient, "e30=",

test/e2e/clusteradm/init_test.go

@@ -30,6 +30,8 @@ var _ = ginkgo.Describe("test clusteradm join with annotations", func() {
 			)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
+			util.WaitClusterManagerApplied(operatorClient)
+
 			ginkgo.By("managedcluster1 join hub with annotations")
 			err = e2e.Clusteradm().Join(
 				"--context", e2e.Cluster().ManagedCluster1().Context(),

test/e2e/clusteradm/joinhubscenario_annotate_test.go

@@ -28,6 +28,8 @@ var _ = ginkgo.Describe("test clusteradm with bootstrap token in singleton mode"
 			)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
+			util.WaitClusterManagerApplied(operatorClient)
+
 			ginkgo.By("managedcluster1 join hub")
 			err = e2e.Clusteradm().Join(
 				"--context", e2e.Cluster().ManagedCluster1().Context(),

test/e2e/clusteradm/joinhubscenario_bootstraptoken_singleton_test.go

@@ -4,6 +4,7 @@ package clusteradme2e
 import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	"open-cluster-management.io/clusteradm/test/e2e/util"
 )
 
 var _ = ginkgo.Describe("test clusteradm with bootstrap token", func() {
@@ -26,6 +27,8 @@ var _ = ginkgo.Describe("test clusteradm with bootstrap token", func() {
 			)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
+			util.WaitClusterManagerApplied(operatorClient)
+
 			ginkgo.By("managedcluster1 join hub")
 			err = e2e.Clusteradm().Join(
 				"--context", e2e.Cluster().ManagedCluster1().Context(),

test/e2e/clusteradm/joinhubscenario_bootstrsptoken_test.go

@@ -4,6 +4,7 @@ package clusteradme2e
 import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	"open-cluster-management.io/clusteradm/test/e2e/util"
 )
 
 var _ = ginkgo.Describe("test clusteradm with service account", func() {
@@ -25,6 +26,8 @@ var _ = ginkgo.Describe("test clusteradm with service account", func() {
 			)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
+			util.WaitClusterManagerApplied(operatorClient)
+
 			ginkgo.By("managedcluster1 join hub")
 			err = e2e.Clusteradm().Join(
 				"--context", e2e.Cluster().ManagedCluster1().Context(),

test/e2e/clusteradm/joinhubscenario_sa_test.go

@@ -4,6 +4,7 @@ package clusteradme2e
 import (
 	"github.com/onsi/ginkgo/v2"
 	"github.com/onsi/gomega"
+	"open-cluster-management.io/clusteradm/test/e2e/util"
 )
 
 var _ = ginkgo.Describe("test clusteradm with timeout", func() {
@@ -25,6 +26,8 @@ var _ = ginkgo.Describe("test clusteradm with timeout", func() {
 			)
 			gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
 
+			util.WaitClusterManagerApplied(operatorClient)
+
 			ginkgo.By("managedcluster1 join hub")
 			err = e2e.Clusteradm().Join(
 				"--context", e2e.Cluster().ManagedCluster1().Context(),

test/e2e/clusteradm/joinhubscenario_timeout_test.go

@@ -8,17 +8,19 @@ import (
 	"os"
 	"time"
 
-	"k8s.io/apimachinery/pkg/api/meta"
-	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
-	"open-cluster-management.io/clusteradm/pkg/config"
-
+	"github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clusterclient "open-cluster-management.io/api/client/cluster/clientset/versioned"
+	operatorclient "open-cluster-management.io/api/client/operator/clientset/versioned"
+	clusterapiv1 "open-cluster-management.io/api/cluster/v1"
+	operatorv1 "open-cluster-management.io/api/operator/v1"
+	"open-cluster-management.io/clusteradm/pkg/config"
 )
 
 // WaitNamespaceDeleted receive a kubeconfigpath, a context name and a namespace name,
@@ -98,6 +100,32 @@ func DeleteClusterFinalizers(restcfg *rest.Config) error {
 	return nil
 }
 
+func WaitClustersDeleted(restcfg *rest.Config) error {
+	clientset, err := clusterclient.NewForConfig(restcfg)
+	if err != nil {
+		return err
+	}
+
+	gomega.Eventually(func() error {
+		clusterList, err := clientset.ClusterV1().ManagedClusters().List(context.TODO(), metav1.ListOptions{})
+		if errors.IsNotFound(err) || len(clusterList.Items) == 0 {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		for _, mcl := range clusterList.Items {
+			err = clientset.ClusterV1().ManagedClusters().Delete(context.TODO(), mcl.Name, metav1.DeleteOptions{})
+			if err != nil {
+				return err
+			}
+		}
+		return fmt.Errorf("wait all clusters are deleted")
+	}, time.Second*120, time.Second*2).Should(gomega.Succeed())
+
+	return nil
+}
+
 // buildConfigFromFlags build rest config for specified context in the kubeconfigfile.
 func buildConfigFromFlags(context, kubeconfigPath string) (*rest.Config, error) {
 	return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
@@ -129,3 +157,24 @@ func NewTestImagePullCredentialFile(fileName string) string {
 func CleanupTestImagePullCredentialFile(fileName string) {
 	_ = os.Remove(fileName)
 }
+
+func WaitClusterManagerApplied(operatorClient operatorclient.Interface) {
+	gomega.Eventually(func() error {
+		cm, err := operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(), "cluster-manager", metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		if !meta.IsStatusConditionTrue(cm.Status.Conditions, operatorv1.ConditionClusterManagerApplied) {
+			return fmt.Errorf("cluster manager is not applied")
+		}
+
+		con := meta.FindStatusCondition(cm.Status.Conditions, operatorv1.ConditionHubRegistrationDegraded)
+		if con == nil {
+			return fmt.Errorf("hub registration is not degraded")
+		}
+		if con.Status != metav1.ConditionFalse || con.Reason != operatorv1.ReasonRegistrationFunctional {
+			return fmt.Errorf("hub registration is not functional")
+		}
+		return nil
+	}, time.Second*60, time.Second*2).Should(gomega.Succeed())
+}