Skip to content

Commit 6d7350f

Browse files
support enable grpc registration when clusteradm init (#519)
Signed-off-by: Zhiwei Yin <[email protected]> Co-authored-by: Zhiwei Yin <[email protected]>
1 parent af5fa02 commit 6d7350f

File tree

13 files changed

+155
-71
lines changed

13 files changed

+155
-71
lines changed

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ require (
3131
open-cluster-management.io/api v1.1.0
3232
open-cluster-management.io/cluster-proxy v0.7.0
3333
open-cluster-management.io/managed-serviceaccount v0.8.0
34-
open-cluster-management.io/ocm v1.1.0
34+
open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7
3535
open-cluster-management.io/sdk-go v1.1.0
3636
sigs.k8s.io/apiserver-network-proxy v0.29.0
3737
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -484,8 +484,8 @@ open-cluster-management.io/cluster-proxy v0.7.0 h1:qOok0BIBL6j4mLRArzJdz0gK5nyyn
484484
open-cluster-management.io/cluster-proxy v0.7.0/go.mod h1:6cgnExpuprO7Le7aqf7bI3H7Nvu3YnXBJCIbJ7wsC0s=
485485
open-cluster-management.io/managed-serviceaccount v0.8.0 h1:8+Z142IUqVT/enxXkyb0nzLUL7JaR7dBM2fDtlCA4pM=
486486
open-cluster-management.io/managed-serviceaccount v0.8.0/go.mod h1:eTixwpLA6XkPQARDjze3k0KRjwn6N22eFOEFx8CpB0I=
487-
open-cluster-management.io/ocm v1.1.0 h1:Gu5+LYMHMCNxE1StB2x9SyH1E1K1cNOTdUrp1Id++T8=
488-
open-cluster-management.io/ocm v1.1.0/go.mod h1:vuIzDonz/ypkaLNqXvjOHSwMA29x+e25s01S8Z0j3gc=
487+
open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7 h1:wRA9v3BH1mfxZiMZVgc0Ert2JziNbekUcg4PpPQUrLk=
488+
open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7/go.mod h1:LlEIZdZrQQduPS6HqFKvdadtFjOfVJzBj80/Ur+exP8=
489489
open-cluster-management.io/sdk-go v1.1.0 h1:vYGkoihIVetyVT4ICO7HjoUHsnh6Gf+Da4ZSmWCamhc=
490490
open-cluster-management.io/sdk-go v1.1.0/go.mod h1:DH4EMNDMiousmaj+noHYQxm48T+dbogiAfALhDnrjMg=
491491
oras.land/oras-go/v2 v2.6.0 h1:X4ELRsiGkrbeox69+9tzTu492FMUu7zJQW6eJU+I2oc=

pkg/cmd/init/cmd.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ var example = `
1616
%[1]s init
1717
1818
# Initialize the hub cluster with the type of authentication. Either or both of csr,awsirsa
19-
%[1]s init --registration-drivers "awsirsa,csr"
19+
%[1]s init --registration-drivers "awsirsa,csr,grpc"
2020
--hubClusterArn arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1
2121
--aws-resource-tags product:v1:tenant:app-name=My-App,product:v1:tenant:created-by=Team-1
2222
--auto-approved-csr-identities="user1,user2"
@@ -89,7 +89,7 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
8989
o.Helm.AddFlags(singletonSet)
9090
cmd.Flags().AddFlagSet(singletonSet)
9191
cmd.Flags().StringSliceVar(&o.registrationDrivers, "registration-drivers", []string{},
92-
"The type of authentication to use for registering and authenticating with hub. Only csr and awsirsa are accepted as valid inputs. This flag can be repeated to specify multiple authentication types.")
92+
"The type of authentication to use for registering and authenticating with hub. Only csr, awsirsa and grpc are accepted as valid inputs. This flag can be repeated to specify multiple authentication types.")
9393
cmd.Flags().StringVar(&o.hubClusterArn, "hub-cluster-arn", "",
9494
"The hubCluster ARN to be passed if awsirsa is one of the registrationAuths and the cluster name in EKS kubeconfig doesn't contain hubClusterArn")
9595
cmd.Flags().StringSliceVar(&o.awsResourceTags, "aws-resource-tags", []string{},
@@ -100,6 +100,8 @@ func NewCmd(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, stream
100100
cmd.Flags().StringSliceVar(&o.autoApprovedARNPatterns, "auto-approved-arn-patterns", []string{},
101101
"List of AWS EKS ARN patterns so any EKS clusters with these patterns will be auto accepted to join with hub cluster")
102102
cmd.Flags().BoolVar(&o.enableSyncLabels, "enable-sync-labels", false, "If true, sync the labels from clustermanager to all hub resources.")
103-
103+
cmd.Flags().StringVar(&o.grpcServer, "grpc-server", "", "The gRPC server address of the hub")
104+
cmd.Flags().StringSliceVar(&o.autoApprovedGRPCIdentities, "auto-approved-grpc-identities", []string{},
105+
"List of users or identities that are accepted and whatever matches can be auto accepted to join hub for grpc clusters")
104106
return cmd
105107
}

pkg/cmd/init/exec.go

Lines changed: 52 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ var (
4040
releaseName = "multicluster-controlplane"
4141
)
4242

43+
var validRegistrationDriver = sets.New[string](operatorv1.CSRAuthType, operatorv1.AwsIrsaAuthType, operatorv1.GRPCAuthType)
44+
4345
func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
4446
klog.V(1).InfoS("init options:", "dry-run", o.ClusteradmFlags.DryRun, "force", o.force, "output-file", o.outputFile)
4547

@@ -95,6 +97,25 @@ func (o *Options) complete(cmd *cobra.Command, args []string) (err error) {
9597
genericclioptionsclusteradm.HubMutableFeatureGate, ocmfeature.DefaultHubAddonManagerFeatureGates),
9698
},
9799
}
100+
if sets.New[string](o.registrationDrivers...).Has(operatorv1.GRPCAuthType) {
101+
if o.grpcServer == "" {
102+
return fmt.Errorf("grpc server should not be empty if registration driver has grpc type")
103+
}
104+
105+
o.clusterManagerChartConfig.ClusterManager.ServerConfiguration = operatorv1.ServerConfiguration{
106+
EndpointsExposure: []operatorv1.EndpointExposure{
107+
{
108+
Protocol: operatorv1.GRPCAuthType,
109+
GRPC: &operatorv1.Endpoint{
110+
Type: operatorv1.EndpointTypeHostname,
111+
Hostname: &operatorv1.HostnameConfig{
112+
Host: o.grpcServer,
113+
},
114+
},
115+
},
116+
},
117+
}
118+
}
98119
o.clusterManagerChartConfig.CreateBootstrapToken = o.useBootstrapToken
99120

100121
if o.imagePullCredFile != "" {
@@ -155,25 +176,29 @@ func (o *Options) validate() error {
155176
return fmt.Errorf("registry should not be empty")
156177
}
157178

158-
validRegistrationDriver := sets.New[string]("csr", "awsirsa")
159179
for _, driver := range o.registrationDrivers {
160180
if !validRegistrationDriver.Has(driver) {
161-
return fmt.Errorf("only csr and awsirsa are valid drivers")
181+
return fmt.Errorf("only csr,awsirsa and grpc are valid drivers")
162182
}
163183
}
164184

165185
if genericclioptionsclusteradm.HubMutableFeatureGate.Enabled("ManagedClusterAutoApproval") {
166186
// If hub registration does not accept awsirsa, we stop user if they also pass in a list of patterns for AWS EKS ARN.
167187

168-
if len(o.autoApprovedARNPatterns) > 0 && !sets.New[string](o.registrationDrivers...).Has("awsirsa") {
188+
if len(o.autoApprovedARNPatterns) > 0 && !sets.New[string](o.registrationDrivers...).Has(operatorv1.AwsIrsaAuthType) {
169189
return fmt.Errorf("should not provide list of patterns for aws eks arn if not initializing hub with awsirsa registration")
170190
}
171191

172192
// If hub registration does not accept csr, we stop user if they also pass in a list of users for CSR auto approval.
173-
if len(o.autoApprovedCSRIdentities) > 0 && !sets.New[string](o.registrationDrivers...).Has("csr") {
193+
if len(o.autoApprovedCSRIdentities) > 0 && !sets.New[string](o.registrationDrivers...).Has(operatorv1.CSRAuthType) {
174194
return fmt.Errorf("should not provide list of users for csr to auto approve if not initializing hub with csr registration")
175195
}
176-
} else if len(o.autoApprovedARNPatterns) > 0 || len(o.autoApprovedCSRIdentities) > 0 {
196+
197+
if len(o.autoApprovedGRPCIdentities) > 0 && !sets.New[string](o.registrationDrivers...).Has(operatorv1.GRPCAuthType) {
198+
return fmt.Errorf("should not provide list of users or identities for grpc cluster to auto approve if not initializing hub with grpc registration")
199+
}
200+
201+
} else if len(o.autoApprovedARNPatterns) > 0 || len(o.autoApprovedCSRIdentities) > 0 || len(o.autoApprovedGRPCIdentities) > 0 {
177202
return fmt.Errorf("should enable feature gate ManagedClusterAutoApproval before passing list of identities")
178203
}
179204

@@ -394,22 +419,38 @@ func (o *Options) deploySingletonControlplane(kubeClient kubernetes.Interface) e
394419

395420
func getRegistrationDrivers(o *Options) ([]operatorv1.RegistrationDriverHub, error) {
396421
registrationDrivers := []operatorv1.RegistrationDriverHub{}
397-
var registrationDriver operatorv1.RegistrationDriverHub
398422

399423
for _, driver := range o.registrationDrivers {
400-
if driver == "csr" {
401-
csr := &operatorv1.CSRConfig{AutoApprovedIdentities: o.autoApprovedCSRIdentities}
402-
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: driver, CSR: csr}
403-
} else if driver == "awsirsa" {
424+
var registrationDriver operatorv1.RegistrationDriverHub
425+
switch driver {
426+
case operatorv1.CSRAuthType:
427+
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: operatorv1.CSRAuthType}
428+
if len(o.autoApprovedCSRIdentities) != 0 {
429+
registrationDriver.CSR = &operatorv1.CSRConfig{
430+
AutoApprovedIdentities: o.autoApprovedCSRIdentities,
431+
}
432+
}
433+
case operatorv1.AwsIrsaAuthType:
404434
hubClusterArn, err := getHubClusterArn(o)
405435
if err != nil {
406436
return registrationDrivers, err
407437
}
408438
awsirsa := &operatorv1.AwsIrsaConfig{HubClusterArn: hubClusterArn, Tags: o.awsResourceTags, AutoApprovedIdentities: o.autoApprovedARNPatterns}
409-
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: driver, AwsIrsa: awsirsa}
439+
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: operatorv1.AwsIrsaAuthType, AwsIrsa: awsirsa}
440+
case operatorv1.GRPCAuthType:
441+
registrationDriver = operatorv1.RegistrationDriverHub{AuthType: operatorv1.GRPCAuthType}
442+
if len(o.autoApprovedGRPCIdentities) != 0 {
443+
registrationDriver.GRPC = &operatorv1.GRPCRegistrationConfig{
444+
AutoApprovedIdentities: o.autoApprovedGRPCIdentities,
445+
}
446+
}
447+
default:
448+
return registrationDrivers, fmt.Errorf("unknown registration-drivers type: %s", driver)
410449
}
450+
411451
registrationDrivers = append(registrationDrivers, registrationDriver)
412452
}
453+
413454
return registrationDrivers, nil
414455
}
415456

pkg/cmd/init/options.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@ type Options struct {
6767
awsResourceTags []string
6868
// enableSyncLabels is to enable the feature which can sync the labels from clustermanager to all hub resources.
6969
enableSyncLabels bool
70+
71+
// grpcServer is the gRPC server of the hub.
72+
grpcServer string
73+
// autoApprovedGRPCIdentities are a list of users or identities that are accepted and whatever matches can
74+
// be auto accepted to join hub for grpc clusters.
75+
autoApprovedGRPCIdentities []string
7076
}
7177

7278
func newOptions(clusteradmFlags *genericclioptionsclusteradm.ClusteradmFlags, streams genericiooptions.IOStreams) *Options {

test/e2e/clusteradm/joinhubscenario_grpc_test.go

Lines changed: 13 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -38,58 +38,27 @@ var _ = ginkgo.Describe("test clusteradm join with grpc", ginkgo.Label("join-hub
3838
ginkgo.By("init hub")
3939
err = e2e.Clusteradm().Init(
4040
"--context", e2e.Cluster().Hub().Context(),
41+
"--registration-drivers", "csr,grpc",
42+
"--grpc-server", "cluster-manager-grpc-server.open-cluster-management-hub.svc:8090",
43+
"--feature-gates=ManagedClusterAutoApproval=true",
44+
"--auto-approved-grpc-identities", "system:serviceaccount:open-cluster-management:agent-registration-bootstrap",
4145
"--bundle-version=latest",
4246
)
4347
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "clusteradm init error")
48+
util.WaitClusterManagerApplied(operatorClient)
4449

45-
ginkgo.By("wait for cluster-manager CR to be created and update cluster-manager to enable grpc")
4650
var clusterManager *operatorv1.ClusterManager
4751
gomega.Eventually(func() error {
48-
clusterManager, err = operatorClient.OperatorV1().ClusterManagers().Get(
49-
context.TODO(), "cluster-manager", metav1.GetOptions{})
50-
if err != nil {
51-
return err
52-
}
53-
54-
// Enable ManagedClusterAutoApproval feature gate
55-
if clusterManager.Spec.RegistrationConfiguration == nil {
56-
clusterManager.Spec.RegistrationConfiguration = &operatorv1.RegistrationHubConfiguration{}
57-
}
58-
clusterManager.Spec.RegistrationConfiguration.FeatureGates = append(
59-
clusterManager.Spec.RegistrationConfiguration.FeatureGates,
60-
operatorv1.FeatureGate{
61-
Feature: "ManagedClusterAutoApproval",
62-
Mode: operatorv1.FeatureGateModeTypeEnable,
63-
},
64-
)
65-
66-
// Add grpc authType in registrationDrivers
67-
clusterManager.Spec.RegistrationConfiguration.RegistrationDrivers = []operatorv1.RegistrationDriverHub{
68-
{
69-
AuthType: operatorv1.GRPCAuthType,
70-
GRPC: &operatorv1.GRPCRegistrationConfig{
71-
AutoApprovedIdentities: []string{
72-
"system:serviceaccount:open-cluster-management:agent-registration-bootstrap",
73-
},
74-
},
75-
},
76-
}
77-
78-
// Add serverConfiguration with grpc protocol
79-
clusterManager.Spec.ServerConfiguration = &operatorv1.ServerConfiguration{
80-
EndpointsExposure: []operatorv1.EndpointExposure{
81-
{
82-
Protocol: "grpc",
83-
},
84-
},
85-
}
86-
87-
_, err = operatorClient.OperatorV1().ClusterManagers().Update(
88-
context.TODO(), clusterManager, metav1.UpdateOptions{})
52+
clusterManager, err = operatorClient.OperatorV1().ClusterManagers().Get(context.TODO(),
53+
"cluster-manager", metav1.GetOptions{})
8954
return err
90-
}, time.Second*30, time.Second*2).Should(gomega.Succeed())
55+
}, time.Second*60, time.Second*2).Should(gomega.Succeed())
9156

92-
util.WaitClusterManagerApplied(operatorClient)
57+
gomega.Expect(len(clusterManager.Spec.RegistrationConfiguration.RegistrationDrivers)).To(
58+
gomega.Equal(2), "should have 2 registration drivers")
59+
60+
gomega.Expect(clusterManager.Spec.ServerConfiguration.EndpointsExposure[0].Protocol).To(
61+
gomega.Equal("grpc"), "server config endpoint exposure protocol should be grpc")
9362

9463
ginkgo.By(fmt.Sprintf("join hub as managedCluster %s with grpc", e2e.Cluster().Hub().Name()))
9564
err = e2e.Clusteradm().Join(

test/e2e/util/helper.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,9 @@ func WaitClustersDeleted(restcfg *rest.Config) error {
120120
return err
121121
}
122122
}
123-
return fmt.Errorf("wait all clusters are deleted: %v", clusterList.Items)
123+
124+
fmt.Printf("wait for all clusters to be deleted: %+v\n", clusterList.Items)
125+
return fmt.Errorf("not all clusters are deleted")
124126
}, time.Second*300, time.Second*2).Should(gomega.Succeed())
125127

126128
return nil

test/e2e/util/util.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ func initE2E() (*TestE2eConfig, error) {
7373
return err
7474
}
7575

76-
fmt.Println("unjoin managedcluster1...")
76+
fmt.Println("unjoin managedCluster on the spoke cluster...")
7777
err := e2eConf.Clusteradm().Unjoin(
7878
"--context", e2eConf.Cluster().ManagedCluster1().Context(),
7979
"--cluster-name", e2eConf.Cluster().ManagedCluster1().Name(),

vendor/modules.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1270,7 +1270,7 @@ open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versio
12701270
open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versioned/scheme
12711271
open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versioned/typed/authentication/v1alpha1
12721272
open-cluster-management.io/managed-serviceaccount/pkg/generated/clientset/versioned/typed/authentication/v1beta1
1273-
# open-cluster-management.io/ocm v1.1.0
1273+
# open-cluster-management.io/ocm v1.1.1-0.20251105064423-d80ec55608e7
12741274
## explicit; go 1.24.0
12751275
open-cluster-management.io/ocm/deploy/cluster-manager/chart
12761276
open-cluster-management.io/ocm/deploy/klusterlet/chart

vendor/open-cluster-management.io/ocm/deploy/cluster-manager/chart/cluster-manager/templates/cluster_manager.yaml

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)