[Contour Gateway Provisioner] add imagepullsecret for envoy and contour (fixes #7138)

Signed-off-by: flbla <[email protected]>

Author: flbla

changelogs/unreleased/7141-flbla-small.md

@@ -0,0 +1 @@
+add a new flag: --image-pull-secret-name, which allows users to specify a secret in the same namespace as the deployed contour control plane for pulling images from private registries. when set, it's used to pull Envoy and Contour images.

cmd/contour/gatewayprovisioner.go

@@ -41,6 +41,7 @@ func registerGatewayProvisioner(app *kingpin.Application) (*kingpin.CmdClause, *
 		leaderElection:        false,
 		leaderElectionID:      "0d879e31.projectcontour.io",
 		gatewayControllerName: "projectcontour.io/gateway-controller",
+		imagePullSecret:       "",
 	}
 
 	cmd.Flag("contour-image", "The container image used for the managed Contour.").
@@ -58,6 +59,10 @@ func registerGatewayProvisioner(app *kingpin.Application) (*kingpin.CmdClause, *
 		Default(provisionerConfig.gatewayControllerName).
 		StringVar(&provisionerConfig.gatewayControllerName)
 
+	cmd.Flag("image-pull-secret-name", "The image pull secret for the managed Envoy and Contour.").
+		Default(provisionerConfig.imagePullSecret).
+		StringVar(&provisionerConfig.imagePullSecret)
+
 	cmd.Flag("incluster", "Use in cluster configuration.").
 		Default("true").
 		BoolVar(&provisionerConfig.inCluster)
@@ -85,6 +90,10 @@ type gatewayProvisionerConfig struct {
 	// by the gateway provisioner.
 	envoyImage string
 
+	// imagePullSecret is the name of the image pull secret that will be used
+	// to pull the Contour and Envoy images.
+	imagePullSecret string
+
 	// metricsBindAddress is the TCP address that the gateway provisioner should bind to for
 	// serving prometheus metrics. It can be set to "0" to disable the metrics serving.
 	metricsBindAddress string
@@ -171,7 +180,7 @@ func createManager(restConfig *rest.Config, provisionerConfig *gatewayProvisione
 	if _, err := controller.NewGatewayClassController(mgr, provisionerConfig.gatewayControllerName); err != nil {
 		return nil, fmt.Errorf("failed to create gatewayclass controller: %w", err)
 	}
-	if _, err := controller.NewGatewayController(mgr, provisionerConfig.gatewayControllerName, provisionerConfig.contourImage, provisionerConfig.envoyImage); err != nil {
+	if _, err := controller.NewGatewayController(mgr, provisionerConfig.gatewayControllerName, provisionerConfig.contourImage, provisionerConfig.envoyImage, provisionerConfig.imagePullSecret); err != nil {
 		return nil, fmt.Errorf("failed to create gateway controller: %w", err)
 	}
 	return mgr, nil

internal/provisioner/controller/gateway.go

@@ -49,15 +49,17 @@ type gatewayReconciler struct {
 	gatewayController gatewayapi_v1.GatewayController
 	contourImage      string
 	envoyImage        string
+	imagePullSecret   string
 	client            client.Client
 	log               logr.Logger
 }
 
-func NewGatewayController(mgr manager.Manager, gatewayController, contourImage, envoyImage string) (controller.Controller, error) {
+func NewGatewayController(mgr manager.Manager, gatewayController, contourImage, envoyImage, imagePullSecret string) (controller.Controller, error) {
 	r := &gatewayReconciler{
 		gatewayController: gatewayapi_v1.GatewayController(gatewayController),
 		contourImage:      contourImage,
 		envoyImage:        envoyImage,
+		imagePullSecret:   imagePullSecret,
 		client:            mgr.GetClient(),
 		log:               ctrl.Log.WithName("gateway-controller"),
 	}
@@ -432,8 +434,8 @@ func (r *gatewayReconciler) ensureContour(ctx context.Context, contour *model.Co
 
 	handleResult("contour config", contourconfig.EnsureContourConfig(ctx, r.client, contour))
 	handleResult("xDS TLS secrets", secret.EnsureXDSSecrets(ctx, r.client, contour, r.contourImage))
-	handleResult("deployment", deployment.EnsureDeployment(ctx, r.client, contour, r.contourImage))
-	handleResult("envoy data plane", dataplane.EnsureDataPlane(ctx, r.client, contour, r.contourImage, r.envoyImage))
+	handleResult("deployment", deployment.EnsureDeployment(ctx, r.client, contour, r.contourImage, r.imagePullSecret))
+	handleResult("envoy data plane", dataplane.EnsureDataPlane(ctx, r.client, contour, r.contourImage, r.envoyImage, r.imagePullSecret))
 	handleResult("contour service", service.EnsureContourService(ctx, r.client, contour))
 
 	switch contour.Spec.NetworkPublishing.Envoy.Type {

internal/provisioner/equality/equality_test.go

@@ -30,10 +30,11 @@ import (
 )
 
 var (
-	testName  = "test"
-	testNs    = testName + "-ns"
-	testImage = "test-image:main"
-	cntr      = &model.Contour{
+	testName            = "test"
+	testNs              = testName + "-ns"
+	testImage           = "test-image:main"
+	testImagePullSecret = ""
+	cntr                = &model.Contour{
 		ObjectMeta: meta_v1.ObjectMeta{
 			Name:      testName,
 			Namespace: testNs,
@@ -122,7 +123,7 @@ func TestDaemonSetConfigChanged(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			original := dataplane.DesiredDaemonSet(cntr, testImage, testImage)
+			original := dataplane.DesiredDaemonSet(cntr, testImage, testImage, testImagePullSecret)
 
 			mutated := original.DeepCopy()
 			tc.mutate(mutated)
@@ -218,7 +219,7 @@ func TestDeploymentConfigChanged(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		original := deployment.DesiredDeployment(cntr, testImage)
+		original := deployment.DesiredDeployment(cntr, testImage, testImagePullSecret)
 		mutated := original.DeepCopy()
 		tc.mutate(mutated)
 		if updated, changed := equality.DeploymentConfigChanged(original, mutated); changed != tc.expect {

internal/provisioner/objects/dataplane/dataplane.go

@@ -75,11 +75,11 @@ var defContainerResources = core_v1.ResourceRequirements{
 }
 
 // EnsureDataPlane ensures an Envoy data plane (daemonset or deployment) exists for the given contour.
-func EnsureDataPlane(ctx context.Context, cli client.Client, contour *model.Contour, contourImage, envoyImage string) error {
+func EnsureDataPlane(ctx context.Context, cli client.Client, contour *model.Contour, contourImage, envoyImage, imagePullSecret string) error {
 	switch contour.Spec.EnvoyWorkloadType {
 	// If a Deployment was specified, provision a Deployment.
 	case model.WorkloadTypeDeployment:
-		desired := desiredDeployment(contour, contourImage, envoyImage)
+		desired := desiredDeployment(contour, contourImage, envoyImage, imagePullSecret)
 
 		updater := func(ctx context.Context, cli client.Client, current, desired *apps_v1.Deployment) error {
 			differ := equality.DeploymentSelectorsDiffer(current, desired)
@@ -94,7 +94,7 @@ func EnsureDataPlane(ctx context.Context, cli client.Client, contour *model.Cont
 
 	// The default workload type is a DaemonSet.
 	default:
-		desired := DesiredDaemonSet(contour, contourImage, envoyImage)
+		desired := DesiredDaemonSet(contour, contourImage, envoyImage, imagePullSecret)
 
 		updater := func(ctx context.Context, cli client.Client, current, desired *apps_v1.DaemonSet) error {
 			differ := equality.DaemonSetSelectorsDiffer(current, desired)
@@ -335,7 +335,7 @@ func desiredContainers(contour *model.Contour, contourImage, envoyImage string)
 // DesiredDaemonSet returns the desired DaemonSet for the provided contour using
 // contourImage as the shutdown-manager/envoy-initconfig container images and
 // envoyImage as Envoy's container image.
-func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *apps_v1.DaemonSet {
+func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage, imagePullSecret string) *apps_v1.DaemonSet {
 	initContainers, containers := desiredContainers(contour, contourImage, envoyImage)
 
 	ds := &apps_v1.DaemonSet{
@@ -403,10 +403,18 @@ func DesiredDaemonSet(contour *model.Contour, contourImage, envoyImage string) *
 		ds.Spec.Template.Spec.Tolerations = contour.Spec.NodePlacement.Envoy.Tolerations
 	}
 
+	if imagePullSecret != "" {
+		ds.Spec.Template.Spec.ImagePullSecrets = []core_v1.LocalObjectReference{
+			{
+				Name: imagePullSecret,
+			},
+		}
+	}
+
 	return ds
 }
 
-func desiredDeployment(contour *model.Contour, contourImage, envoyImage string) *apps_v1.Deployment {
+func desiredDeployment(contour *model.Contour, contourImage, envoyImage, imagePullSecret string) *apps_v1.Deployment {
 	initContainers, containers := desiredContainers(contour, contourImage, envoyImage)
 
 	deployment := &apps_v1.Deployment{
@@ -488,6 +496,14 @@ func desiredDeployment(contour *model.Contour, contourImage, envoyImage string)
 		deployment.Spec.Template.Spec.Tolerations = contour.Spec.NodePlacement.Envoy.Tolerations
 	}
 
+	if imagePullSecret != "" {
+		deployment.Spec.Template.Spec.ImagePullSecrets = []core_v1.LocalObjectReference{
+			{
+				Name: imagePullSecret,
+			},
+		}
+	}
+
 	return deployment
 }
 

internal/provisioner/objects/dataplane/dataplane_test.go

@@ -311,6 +311,7 @@ func TestDesiredDaemonSet(t *testing.T) {
 
 	testContourImage := "ghcr.io/projectcontour/contour:test"
 	testEnvoyImage := "docker.io/envoyproxy/envoy:test"
+	testImagePullSecret := ""
 	testLogLevelArg := "--log-level debug"
 	testBaseIDArg := "--base-id 1"
 	testEnvoyMaxHeapSize := "--overload-max-heap=8000000000"
@@ -343,7 +344,7 @@ func TestDesiredDaemonSet(t *testing.T) {
 	cntr.Spec.EnvoyMaxHeapSizeBytes = 8000000000
 	cntr.Spec.EnvoyMaxDownstreamConnections = 42
 
-	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage)
+	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
 	container := checkDaemonSetHasContainer(t, ds, EnvoyContainerName, true)
 	checkContainerHasArg(t, container, testLogLevelArg)
 	checkContainerHasArg(t, container, testBaseIDArg)
@@ -386,7 +387,8 @@ func TestDesiredDeployment(t *testing.T) {
 
 	testContourImage := "ghcr.io/projectcontour/contour:test"
 	testEnvoyImage := "docker.io/envoyproxy/envoy:test"
-	deploy := desiredDeployment(cntr, testContourImage, testEnvoyImage)
+	testImagePullSecret := ""
+	deploy := desiredDeployment(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
 	checkDeploymentHasStrategy(t, deploy, cntr.Spec.EnvoyDeploymentStrategy)
 	checkEnvoyDeploymentHasAffinity(t, deploy, cntr)
 }
@@ -414,7 +416,8 @@ func TestNodePlacementDaemonSet(t *testing.T) {
 
 	testContourImage := "ghcr.io/projectcontour/contour:test"
 	testEnvoyImage := "docker.io/envoyproxy/envoy:test"
-	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage)
+	testImagePullSecret := ""
+	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
 	checkDaemonSetHasNodeSelector(t, ds, selectors)
 	checkDaemonSetHasTolerations(t, ds, tolerations)
 }
@@ -436,9 +439,28 @@ func TestEnvoyCustomPorts(t *testing.T) {
 
 	testContourImage := "ghcr.io/projectcontour/contour:test"
 	testEnvoyImage := "docker.io/envoyproxy/envoy:test"
-	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage)
+	testImagePullSecret := ""
+	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
 	checkContainerHasPort(t, ds, int32(metricPort))
 
 	container := checkDaemonSetHasContainer(t, ds, EnvoyContainerName, true)
 	checkContainerHasReadinessPort(t, container, 8020)
 }
+
+func TestDesiredDaemonSetWithImagePullSecret(t *testing.T) {
+	name := "ds-test"
+	cntr := model.Default(fmt.Sprintf("%s-ns", name), name)
+	cntr.Spec.NetworkPublishing.Envoy.Ports = []model.Port{
+		{Name: "http", ServicePort: 80, ContainerPort: 8080},
+	}
+	testContourImage := "ghcr.io/projectcontour/contour:test"
+	testEnvoyImage := "docker.io/envoyproxy/envoy:test"
+	testImagePullSecret := "my-secret"
+
+	ds := DesiredDaemonSet(cntr, testContourImage, testEnvoyImage, testImagePullSecret)
+
+	require.NotNil(t, ds)
+	require.NotNil(t, ds.Spec.Template.Spec.ImagePullSecrets)
+	require.Len(t, ds.Spec.Template.Spec.ImagePullSecrets, 1)
+	require.Equal(t, testImagePullSecret, ds.Spec.Template.Spec.ImagePullSecrets[0].Name)
+}

internal/provisioner/objects/deployment/deployment.go

@@ -52,8 +52,8 @@ const (
 )
 
 // EnsureDeployment ensures a deployment using image exists for the given contour.
-func EnsureDeployment(ctx context.Context, cli client.Client, contour *model.Contour, image string) error {
-	desired := DesiredDeployment(contour, image)
+func EnsureDeployment(ctx context.Context, cli client.Client, contour *model.Contour, image, imagePullSecret string) error {
+	desired := DesiredDeployment(contour, image, imagePullSecret)
 
 	updater := func(ctx context.Context, cli client.Client, current, desired *apps_v1.Deployment) error {
 		differ := equality.DeploymentSelectorsDiffer(current, desired)
@@ -82,7 +82,7 @@ func EnsureDeploymentDeleted(ctx context.Context, cli client.Client, contour *mo
 
 // DesiredDeployment returns the desired deployment for the provided contour using
 // image as Contour's container image.
-func DesiredDeployment(contour *model.Contour, image string) *apps_v1.Deployment {
+func DesiredDeployment(contour *model.Contour, image, imagePullSecret string) *apps_v1.Deployment {
 	xdsPort := objects.XDSPort
 	args := []string{
 		"serve",
@@ -276,6 +276,14 @@ func DesiredDeployment(contour *model.Contour, image string) *apps_v1.Deployment
 		deploy.Spec.Template.Spec.Tolerations = contour.Spec.NodePlacement.Contour.Tolerations
 	}
 
+	if imagePullSecret != "" {
+		deploy.Spec.Template.Spec.ImagePullSecrets = []core_v1.LocalObjectReference{
+			{
+				Name: imagePullSecret,
+			},
+		}
+	}
+
 	return deploy
 }
 

internal/provisioner/objects/deployment/deployment_test.go

@@ -178,7 +178,8 @@ func TestDesiredDeployment(t *testing.T) {
 	}
 
 	testContourImage := "ghcr.io/projectcontour/contour:test"
-	deploy := DesiredDeployment(cntr, testContourImage)
+	testImagePullSecret := ""
+	deploy := DesiredDeployment(cntr, testContourImage, testImagePullSecret)
 
 	container := checkDeploymentHasContainer(t, deploy, contourContainerName, true)
 	checkContainerHasImage(t, container, testContourImage)
@@ -239,7 +240,7 @@ func TestDesiredDeploymentWhenSettingWatchNamespaces(t *testing.T) {
 			cntr.Spec.IngressClassName = &icName
 			// Change the Contour watch namespaces flag
 			cntr.Spec.WatchNamespaces = tc.namespaces
-			deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test")
+			deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test", "")
 			container := checkDeploymentHasContainer(t, deploy, contourContainerName, true)
 			arg := fmt.Sprintf("--watch-namespaces=%s", strings.Join(append(model.NamespacesToStrings(tc.namespaces), cntr.Namespace), ","))
 			checkContainerHasArg(t, container, arg)
@@ -268,7 +269,7 @@ func TestNodePlacementDeployment(t *testing.T) {
 		},
 	}
 
-	deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test")
+	deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test", "")
 
 	checkDeploymentHasNodeSelector(t, deploy, selectors)
 	checkDeploymentHasTolerations(t, deploy, tolerations)
@@ -297,7 +298,7 @@ func TestDesiredDeploymentWhenSettingDisabledFeature(t *testing.T) {
 			cntr.Spec.IngressClassName = &icName
 			cntr.Spec.DisabledFeatures = tc.disabledFeatures
 			// Change the Contour watch namespaces flag
-			deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test")
+			deploy := DesiredDeployment(cntr, "ghcr.io/projectcontour/contour:test", "")
 			container := checkDeploymentHasContainer(t, deploy, contourContainerName, true)
 			for _, f := range tc.disabledFeatures {
 				arg := fmt.Sprintf("--disable-feature=%s", string(f))
@@ -306,3 +307,19 @@ func TestDesiredDeploymentWhenSettingDisabledFeature(t *testing.T) {
 		})
 	}
 }
+
+func TestDesiredDeploymentWithImagePullSecret(t *testing.T) {
+	name := "deploy-test"
+	cntr := model.Default(fmt.Sprintf("%s-ns", name), name)
+	icName := "test-ic"
+	cntr.Spec.IngressClassName = &icName
+
+	testContourImage := "ghcr.io/projectcontour/contour:test"
+	testImagePullSecret := "my-secret"
+	deploy := DesiredDeployment(cntr, testContourImage, testImagePullSecret)
+
+	require.NotNil(t, deploy)
+	require.NotNil(t, deploy.Spec.Template.Spec.ImagePullSecrets)
+	require.Len(t, deploy.Spec.Template.Spec.ImagePullSecrets, 1)
+	require.Equal(t, testImagePullSecret, deploy.Spec.Template.Spec.ImagePullSecrets[0].Name)
+}