support join a cluster via grpc (#518)

Signed-off-by: Zhiwei Yin <[email protected]>
Co-authored-by: Zhiwei Yin <[email protected]>

Author: openshift-cherrypick-robot

go.mod

@@ -40,6 +40,7 @@ require (
 )
 
 require (
+	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	dario.cat/mergo v1.0.1 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
 	github.com/BurntSushi/toml v1.5.0 // indirect
@@ -53,6 +54,7 @@ require (
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
+	github.com/cloudevents/sdk-go/v2 v2.16.2 // indirect
 	github.com/containerd/containerd v1.7.27 // indirect
 	github.com/containerd/errdefs v0.3.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
@@ -75,6 +77,7 @@ require (
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/gnostic-models v0.6.9 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
+cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
 dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
@@ -38,6 +40,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNSjIRk=
 github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA=
+github.com/cloudevents/sdk-go/v2 v2.16.2 h1:ZYDFrYke4FD+jM8TZTJJO6JhKHzOQl2oqpFK1D+NnQM=
+github.com/cloudevents/sdk-go/v2 v2.16.2/go.mod h1:laOcGImm4nVJEU+PHnUrKL56CKmRL65RlQF0kRmW/kg=
 github.com/containerd/containerd v1.7.27 h1:yFyEyojddO3MIGVER2xJLWoCIn+Up4GaHFquP7hsFII=
 github.com/containerd/containerd v1.7.27/go.mod h1:xZmPnl75Vc+BLGt4MIfu6bp+fy03gdHAn9bz+FreFR0=
 github.com/containerd/errdefs v0.3.0 h1:FSZgGOeK4yuT/+DnF07/Olde/q4KBoMsaamhXxIMDp4=
@@ -308,6 +312,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=

pkg/cmd/accept/exec.go

@@ -28,7 +28,9 @@ const (
 	groupNameBootstrap               = "system:bootstrappers:managedcluster"
 	userNameSignatureBootstrapPrefix = "system:bootstrap:"
 	userNameSignatureSA              = "system:serviceaccount:open-cluster-management:agent-registration-bootstrap"
+	userNameGRPCSignatureSA          = "system:serviceaccount:open-cluster-management-hub:grpc-server-sa"
 	groupNameSA                      = "system:serviceaccounts:open-cluster-management"
+	groupNameGRPC                    = "system:serviceaccounts:open-cluster-management-hub"
 	clusterLabel                     = "open-cluster-management.io/cluster-name"
 )
 
@@ -143,13 +145,15 @@ func (o *Options) approveCSR(kubeClient *kubernetes.Clientset, clusterName strin
 		for _, item := range csrs.Items {
 			// Does not have the correct name prefix
 			if !strings.HasPrefix(item.Spec.Username, userNameSignatureBootstrapPrefix) &&
-				!strings.HasPrefix(item.Spec.Username, userNameSignatureSA) {
+				!strings.HasPrefix(item.Spec.Username, userNameSignatureSA) &&
+				!strings.HasPrefix(item.Spec.Username, userNameGRPCSignatureSA) {
 				continue
 			}
 			// Check groups
 			groups := sets.NewString(item.Spec.Groups...)
 			if !groups.Has(groupNameBootstrap) &&
-				!groups.Has(groupNameSA) {
+				!groups.Has(groupNameSA) &&
+				!groups.Has(groupNameGRPC) {
 				continue
 			}
 			passedCSRs = append(passedCSRs, item)

pkg/cmd/join/cmd.go

@@ -21,6 +21,8 @@ var example = `
 %[1]s join --hub-token <tokenID.tokenSecret> --hub-apiserver <hub_apiserver_url> --cluster-name <cluster_name> --registration-auth awsirsa --hub-cluster-arn arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster-1
 # Join a cluster to the hub and annotate the ManagedCluster
 %[1]s join --hub-token <tokenID.tokenSecret> --hub-apiserver <hub_apiserver_url> --cluster-name <cluster_name> --klusterlet-annotation foo=bar --klusterlet-annotation bar=foo
+# Join a cluster to the hub via gRPC
+%[1]s join --hub-token <tokenID.tokenSecret> --hub-apiserver <hub_apiserver_url> --cluster-name <cluster_name> --registration-auth grpc --grpc-server <grpc_server_address>
 `
 
 // NewCmd ...
@@ -83,10 +85,12 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
 	cmd.Flags().BoolVar(&o.createNameSpace, "create-namespace", true, "If true, create the operator namespace(open-cluster-management) and the agent namespace(open-cluster-management-agent for Default mode, <klusterlet-name> for Hosted mode), otherwise use existing one")
 	cmd.Flags().BoolVar(&o.enableSyncLabels, "enable-sync-labels", false, "If true, sync the labels from klusterlet to all agent resources.")
 	cmd.Flags().Int32Var(&o.clientCertExpirationSeconds, "client-cert-expiration-seconds", 31536000, "clientCertExpirationSeconds represents the seconds of a client certificate to expire.")
-	cmd.Flags().StringVar(&o.registrationAuth, "registration-auth", "", "The type of authentication to use for registering and authenticating with hub")
+	cmd.Flags().StringVar(&o.registrationAuth, "registration-auth", "csr", "The type of authentication to use for registering and authenticating with hub. The supported types including: csr, grpc and awsirsa. The default type is csr.")
 	cmd.Flags().StringVar(&o.hubClusterArn, "hub-cluster-arn", "", "The arn of the hub cluster(i.e. EKS cluster) to which managed-cluster will join")
 	cmd.Flags().StringVar(&o.managedClusterArn, "managed-cluster-arn", "", "The arn of the managed cluster(i.e. EKS cluster) which will be joining the hub")
 	cmd.Flags().StringArrayVar(&o.klusterletAnnotations, "klusterlet-annotation", []string{}, fmt.Sprintf("Annotations to set on the ManagedCluster, in key=value format. Note: each key will be automatically prefixed with '%s/', if not set.", operatorv1.ClusterAnnotationsKeyPrefix))
 	cmd.Flags().StringVar(&o.klusterletValuesFile, "klusterlet-values-file", "", "The path to a YAML file containing klusterlet Helm chart values. The values from the file override both the default klusterlet chart values and the values from other flags.")
+	cmd.Flags().StringVar(&o.grpcServer, "grpc-server", "", "The gRPC server address of the hub")
+	cmd.Flags().StringVar(&o.grpcCAFile, "grpc-ca-file", "", "Path to gRPC server CA PEM; required if --grpc-server is set")
 	return cmd
 }

pkg/cmd/join/exec.go

@@ -36,8 +36,8 @@ import (
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	ocmfeature "open-cluster-management.io/api/feature"
 	operatorv1 "open-cluster-management.io/api/operator/v1"
-
 	"open-cluster-management.io/clusteradm/pkg/cmd/join/preflight"
+	"open-cluster-management.io/clusteradm/pkg/config"
 	genericclioptionsclusteradm "open-cluster-management.io/clusteradm/pkg/genericclioptions"
 	"open-cluster-management.io/clusteradm/pkg/helpers"
 	"open-cluster-management.io/clusteradm/pkg/helpers/klusterlet"
@@ -48,13 +48,14 @@ import (
 	"open-cluster-management.io/clusteradm/pkg/helpers/wait"
 	"open-cluster-management.io/clusteradm/pkg/version"
 	"open-cluster-management.io/ocm/pkg/operator/helpers/chart"
+	"open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/cert"
+	sdkgrpc "open-cluster-management.io/sdk-go/pkg/cloudevents/generic/options/grpc"
 	sdkhelpers "open-cluster-management.io/sdk-go/pkg/helpers"
 )
 
 const (
-	AgentNamespacePrefix = "open-cluster-management-"
-
-	OperatorNamesapce     = "open-cluster-management"
+	AgentNamespacePrefix  = "open-cluster-management-"
+	OperatorNamespace     = "open-cluster-management"
 	DefaultOperatorName   = "klusterlet"
 	AwsIrsaAuthentication = "awsirsa"
 )
@@ -191,7 +192,7 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		if err != nil {
 			return err
 		}
-		o.HubCADate = cabytes
+		o.HubCAData = cabytes
 	}
 
 	// code logic of building hub client in join process:
@@ -212,6 +213,12 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
 		return err
 	}
 
+	if o.grpcServer != "" {
+		if err = o.getGRPCCAData(externalClientUnSecure); err != nil {
+			return err
+		}
+	}
+
 	// get managed cluster externalServerURL
 	var kubeClient *kubernetes.Clientset
 	switch o.mode {
@@ -284,6 +291,11 @@ func (o *Options) validate() error {
 		return err
 	}
 
+	err = o.setGRPCConfig()
+	if err != nil {
+		return err
+	}
+
 	// get ManagedKubeconfig from given file
 	if o.mode == string(operatorv1.InstallModeHosted) {
 		managedConfig, err := os.ReadFile(o.managedKubeconfigFile)
@@ -323,8 +335,19 @@ func (o *Options) validate() error {
 		return err
 	}
 
-	if (o.registrationAuth == AwsIrsaAuthentication) && (o.hubClusterArn == "") {
-		return gherrors.New("hubClusterArn cannot be empty if registrationAuth type is awsirsa")
+	switch o.registrationAuth {
+	case operatorv1.AwsIrsaAuthType:
+		if o.hubClusterArn == "" {
+			return gherrors.New("hub-cluster-arn is required when registration-auth type is awsirsa")
+		}
+	case operatorv1.GRPCAuthType:
+		if o.grpcServer == "" {
+			return gherrors.New("grpc-server is required when registration-auth type is grpc")
+		}
+	case operatorv1.CSRAuthType:
+		// default auth type. do nothing
+	default:
+		return gherrors.New("invalid registration-Auth type")
 	}
 
 	return nil
@@ -395,7 +418,7 @@ func (o *Options) applyKlusterlet(r *reader.ResourceReader, operatorClient opera
 		o.klusterletChartConfig.NoOperator = true
 	}
 
-	crds, raw, err := chart.RenderKlusterletChart(o.klusterletChartConfig, OperatorNamesapce)
+	crds, raw, err := chart.RenderKlusterletChart(o.klusterletChartConfig, OperatorNamespace)
 	if err != nil {
 		return err
 	}
@@ -457,7 +480,7 @@ func checkIfRegistrationOperatorAvailable(f util.Factory) (bool, error) {
 		return false, err
 	}
 
-	deploy, err := client.AppsV1().Deployments(OperatorNamesapce).
+	deploy, err := client.AppsV1().Deployments(OperatorNamespace).
 		Get(context.TODO(), DefaultOperatorName, metav1.GetOptions{})
 	if err != nil {
 		if errors.IsNotFound(err) {
@@ -550,7 +573,7 @@ func waitUntilRegistrationOperatorConditionIsTrue(w io.Writer, f util.Factory, t
 
 	return helpers.WatchUntil(
 		func() (watch.Interface, error) {
-			return client.CoreV1().Pods(OperatorNamesapce).
+			return client.CoreV1().Pods(OperatorNamespace).
 				Watch(context.TODO(), metav1.ListOptions{
 					TimeoutSeconds: &timeout,
 					LabelSelector:  "app=klusterlet",
@@ -664,9 +687,9 @@ func (o *Options) createClientcmdapiv1Config(externalClientUnSecure *kubernetes.
 	bootstrapConfig := bootstrapExternalConfigUnSecure.DeepCopy()
 	bootstrapConfig.Clusters[0].Cluster.InsecureSkipTLSVerify = false
 	bootstrapConfig.Clusters[0].Cluster.Server = o.hubAPIServer
-	if o.HubCADate != nil {
+	if o.HubCAData != nil {
 		// directly set ca-data if --ca-file is set
-		bootstrapConfig.Clusters[0].Cluster.CertificateAuthorityData = o.HubCADate
+		bootstrapConfig.Clusters[0].Cluster.CertificateAuthorityData = o.HubCAData
 	} else {
 		// get ca data from externalClientUnsecure, ca may empty(cluster-info exists with no ca data)
 		ca, err := sdkhelpers.GetCACert(externalClientUnSecure)
@@ -713,6 +736,59 @@ func (o *Options) setKubeconfig() error {
 	return nil
 }
 
+func (o *Options) setGRPCConfig() error {
+	if o.registrationAuth != operatorv1.GRPCAuthType {
+		return nil
+	}
+
+	gRPCConfig := sdkgrpc.GRPCConfig{
+		CertConfig: cert.CertConfig{
+			CAData: cert.Bytes(o.grpcCAData),
+		},
+		URL:   o.grpcServer,
+		Token: o.token,
+	}
+
+	configStr, err := yaml.Marshal(gRPCConfig)
+	if err != nil {
+		return fmt.Errorf("failed to marshal GRPC server configuration. %v", err)
+	}
+
+	o.klusterletChartConfig.GRPCConfig = string(configStr)
+	o.klusterletChartConfig.Klusterlet.RegistrationConfiguration.RegistrationDriver = operatorv1.RegistrationDriver{
+		AuthType: operatorv1.GRPCAuthType,
+	}
+	return nil
+}
+
+func (o *Options) getGRPCCAData(kubeClient kubernetes.Interface) error {
+	if o.grpcCAFile != "" {
+		caData, err := os.ReadFile(o.grpcCAFile)
+		if err != nil {
+			return fmt.Errorf("--grpc-ca-file %q read failed: %w", o.grpcCAFile, err)
+		}
+		if len(caData) == 0 {
+			return fmt.Errorf("--grpc-ca-file %q is empty", o.grpcCAFile)
+		}
+		o.grpcCAData = caData
+		return nil
+	}
+
+	cm, err := kubeClient.CoreV1().ConfigMaps(config.HubClusterNamespace).Get(context.TODO(),
+		config.CABundleConfigMap, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get CA bundle configmap for gRPC server: %w", err)
+	}
+
+	caBundle, ok := cm.Data["ca-bundle.crt"]
+	if !ok || len(strings.TrimSpace(caBundle)) == 0 {
+		return fmt.Errorf("ConfigMap %s/%s is missing or has empty key 'ca-bundle.crt'",
+			config.HubClusterNamespace, config.CABundleConfigMap)
+	}
+	o.grpcCAData = []byte(caBundle)
+	return nil
+}
+
 func mergeCertificateData(caBundles ...[]byte) ([]byte, error) {
 	var all []*x509.Certificate
 	for _, caBundle := range caBundles {

pkg/cmd/join/options.go

@@ -19,8 +19,15 @@ type Options struct {
 	token string
 	// The external hub apiserver url (https://<host>:<port>)
 	hubAPIServer string
+	// The grpc server of the hub cluster
+	grpcServer string
+
 	// The hub ca-file(optional)
 	caFile string
+
+	// The grpc ca file which can be found in the configmap ca-bundle-configmap in open-cluster-management-hub ns
+	grpcCAFile string
+
 	// the name under the cluster must be imported
 	clusterName string
 
@@ -58,13 +65,16 @@ type Options struct {
 	forceManagedInClusterEndpointLookup bool
 	hubInClusterEndpoint                string
 
-	// Values below are tempoary data
-	// HubCADate: data in hub ca file
-	HubCADate []byte
+	// Values below are temporary data
+	// HubCAData: data in hub ca file
+	HubCAData []byte
 	// hub config
 	HubConfig *clientcmdapiv1.Config
 
-	// The URL of a forward proxy server which will be used by agnets on the managed cluster
+	// grpcCAData: ca data used by the GRPC server
+	grpcCAData []byte
+
+	// The URL of a forward proxy server which will be used by agents on the managed cluster
 	// to connect to the hub cluster (optional)
 	proxyURL string
 	// The proxy server ca-file(optional)

pkg/config/env.go

@@ -16,4 +16,5 @@ const (
 	ManagedClusterNamespace           = "open-cluster-management-agent"
 	ManagedProxyConfigurationName     = "cluster-proxy"
 	ImagePullSecret                   = "open-cluster-management-image-pull-credentials"
+	CABundleConfigMap                 = "ca-bundle-configmap"
 )