POC: Additional CA in kcp server

.prow.yaml

@@ -64,9 +64,6 @@ presubmits:
         - image: quay.io/containers/buildah:v1.38.0
           command:
             - hack/build-image.sh
-          env:
-            - name: DRY_RUN
-              value: "1"
           # docker-in-docker needs privileged mode
           securityContext:
             privileged: true

hack/build-image-docker.sh

@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The KCP Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Build container images for KCP using Docker
+#
+# This script builds container images using Docker (with or without buildx).
+#
+# Usage examples:
+#   # Build locally with default settings (uses current git commit hash)
+#   ./hack/build-image-docker.sh
+#   
+#   # Build locally with custom repository name
+#   REPOSITORY=my-registry/kcp ./hack/build-image-docker.sh
+#   
+#   # Build locally without pushing (dry run)
+#   DRY_RUN=1 ./hack/build-image-docker.sh
+#   
+#   # Build for specific architectures only
+#   ARCHITECTURES="amd64" ./hack/build-image-docker.sh
+#
+# Environment variables:
+#   REPOSITORY - Override default repository (default: ghcr.io/kcp-dev/kcp)
+#   ARCHITECTURES - Space-separated list of architectures (default: "amd64 arm64")
+#   DRY_RUN - Set to any value to build locally without pushing
+#   KCP_GHCR_USERNAME/KCP_GHCR_PASSWORD - Registry credentials for pushing
+#
+# Build tool support:
+#   - docker + buildx: Multi-arch support with intelligent platform handling  
+#   - docker only: Single architecture fallback
+
+set -euo pipefail
+
+# make git available
+if ! [ -x "$(command -v git)" ]; then
+  echo "Installing git ..."
+  yum install -y git
+fi
+
+# Check if docker is available
+if ! [ -x "$(command -v docker)" ]; then
+  echo "Error: Docker is not available"
+  exit 1
+fi
+
+echo "Using docker for container builds"
+# Check if buildx is available for multi-arch builds
+if docker buildx version >/dev/null 2>&1; then
+  echo "Docker buildx is available for multi-arch builds"
+  DOCKER_BUILDX=true
+else
+  echo "Docker buildx not available, falling back to single-arch builds"
+  DOCKER_BUILDX=false
+fi
+
+
+if [ -z "${REPOSITORY:-}" ]; then
+  echo "Error: REPOSITORY environment variable is required"
+  exit 1
+fi
+repository=$REPOSITORY
+architectures=${ARCHITECTURES:-"amd64 arm64"}
+
+# when building locally, just tag with the current HEAD hash.
+version="$(git rev-parse --short HEAD)"
+branchName=""
+
+# deduce the tag from the Prow job metadata
+if [ -n "${PULL_BASE_REF:-}" ]; then
+  version="$(git tag --list "$PULL_BASE_REF")"
+
+  if [ -z "$version" ]; then
+    # if the base ref did not point to a tag, it's a branch name
+    version="$(git rev-parse --short "$PULL_BASE_REF")"
+    branchName="$PULL_BASE_REF"
+  else
+    # If PULL_BASE_REF is a tag, there is no branch available locally, plus
+    # there is no guarantee that vX.Y.Z is tagged _only_ in the release-X.Y
+    # branch; because of this we have to deduce the branch name from the tag
+    branchName="$(echo "$version" | sed -E 's/^v([0-9]+)\.([0-9]+)\..*/release-\1.\2/')"
+  fi
+fi
+
+# Prefix with "pr-" if not on a tag or branch
+if [ -n "${PULL_NUMBER:-}" ]; then
+  version="pr-$PULL_NUMBER-$version"
+  repository="$repository-prs"
+fi
+
+image="$repository:$version"
+echo "Building container image $image ..."
+
+# Function to build images with docker buildx
+build_with_docker_buildx() {
+  echo "Building multi-arch image $image ..."
+
+  # Create platforms string for buildx
+  platforms=""
+  for arch in $architectures; do
+    if [ -n "$platforms" ]; then
+      platforms="$platforms,linux/$arch"
+    else
+      platforms="linux/$arch"
+    fi
+  done
+
+  # For push builds, use multi-platform; for local builds, build per arch
+  if [ -z "${DRY_RUN:-}" ]; then
+    # Building for push - use multi-platform with --push
+    docker buildx build \
+      --file Dockerfile \
+      --tag "$image" \
+      --platform "$platforms" \
+      --build-arg "TARGETOS=linux" \
+      --push \
+      .
+  else
+    # For local/dry-run builds, build each architecture separately with --load
+    for arch in $architectures; do
+      fullTag="$image-$arch"
+      echo "Building $fullTag ..."
+      docker buildx build \
+        --file Dockerfile \
+        --tag "$fullTag" \
+        --platform "linux/$arch" \
+        --build-arg "TARGETOS=linux" \
+        --build-arg "TARGETARCH=$arch" \
+        --load \
+        .
+    done
+    # Tag the first architecture as the main image for local use
+    first_arch=$(echo $architectures | cut -d' ' -f1)
+    docker tag "$image-$first_arch" "$image"
+  fi
+}
+
+# Function to build images with regular docker (single arch only)
+build_with_docker() {
+  # Use only the first architecture for regular docker
+  arch=$(echo $architectures | cut -d' ' -f1)
+  fullTag="$image-$arch"
+
+  echo "Building single-arch image $fullTag (docker without buildx) ..."
+  docker build \
+    --file Dockerfile \
+    --tag "$fullTag" \
+    --platform "linux/$arch" \
+    --build-arg "TARGETOS=linux" \
+    --build-arg "TARGETARCH=$arch" \
+    .
+
+  # Tag it as the main image too
+  docker tag "$fullTag" "$image"
+}
+
+# Build images based on available docker features
+if [ "$DOCKER_BUILDX" = true ]; then
+  build_with_docker_buildx
+else
+  build_with_docker
+fi
+
+# Additionally to an image tagged with the Git tag, we also
+# release images tagged with the current branch name, which
+# is somewhere between a blanket "latest" tag and a specific
+# tag.
+if [ -n "$branchName" ] && [ -z "${PULL_NUMBER:-}" ]; then
+  branchImage="$repository:$branchName"
+
+  if [ "$DOCKER_BUILDX" = true ]; then
+    echo "Tagging multi-arch image as $branchImage ..."
+    docker tag "$image" "$branchImage"
+  else
+    echo "Tagging single-arch image as $branchImage ..."
+    docker tag "$image" "$branchImage"
+  fi
+fi
+
+# push images, except in dry runs
+if [ -z "${DRY_RUN:-}" ]; then
+  echo "Logging into GHCR ..."
+
+  if [ "$DOCKER_BUILDX" = true ]; then
+    # buildx with --push already pushed during build
+    echo "Images already pushed during buildx build"
+  else
+    # Regular docker - need to login and push
+    if [ -n "${GHCR_USERNAME:-}" ] && [ -n "${GHCR_PASSWORD:-}" ]; then
+      echo "$GHCR_PASSWORD" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+    else
+      echo "Skipping login (GHCR_USERNAME/GHCR_PASSWORD not provided)"
+    fi
+
+    echo "Pushing images ..."
+    docker push "$image"
+
+    if [ -n "${branchImage:-}" ]; then
+      docker push "$branchImage"
+
+
+    fi
+  fi
+else
+  echo "Not pushing images because \$DRY_RUN is set."
+fi
+
+echo "Done."

pkg/server/config.go

@@ -136,6 +136,9 @@ type ExtraConfig struct {
 	CacheDiscoveringDynamicSharedInformerFactory *informer.DiscoveringDynamicSharedInformerFactory
 	CacheKcpSharedInformerFactory                kcpinformers.SharedInformerFactory
 	CacheKubeSharedInformerFactory               kcpkubernetesinformers.SharedInformerFactory
+
+	// If CABundleFile is set, read its contents into CABundleData
+	CABundleData []byte
 }
 
 type completedConfig struct {
@@ -186,6 +189,14 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co
 		Options: opts,
 	}
 
+	if opts.Extra.CABundleFile != "" {
+		data, err := os.ReadFile(opts.Extra.CABundleFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read ca-bundle-file %q: %w", opts.Extra.CABundleFile, err)
+		}
+		c.CABundleData = data
+	}
+
 	if opts.Extra.ProfilerAddress != "" {
 		//nolint:errcheck,gosec
 		go http.ListenAndServe(opts.Extra.ProfilerAddress, nil)
@@ -323,6 +334,15 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co
 		if err != nil {
 			return nil, fmt.Errorf("failed to load the external logical cluster admin kubeconfig from %q: %w", c.Options.Extra.ExternalLogicalClusterAdminKubeconfig, err)
 		}
+		if c.CABundleData != nil {
+			// Inject CA bundle into rest.Config
+			if c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData == nil {
+				c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData = c.CABundleData
+			} else {
+				// Append to existing CA data
+				c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData = append(c.ExternalLogicalClusterAdminConfig.TLSClientConfig.CAData, c.CABundleData...)
+			}
+		}
 	}
 
 	// Setup apiextensions * informers
@@ -345,12 +365,12 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co
 		return nil, err
 	}
 	var userToken string
-	if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.Admin) {
+	if sets.New(opts.Extra.BatteriesIncluded...).Has(batteries.Admin) {
 		c.kcpAdminToken, c.shardAdminToken, userToken, c.shardAdminTokenHash, err = opts.AdminAuthentication.ApplyTo(c.GenericConfig)
 		if err != nil {
 			return nil, err
 		}
-		if sets.New[string](opts.Extra.BatteriesIncluded...).Has(batteries.User) {
+		if sets.New(opts.Extra.BatteriesIncluded...).Has(batteries.User) {
 			c.userToken = userToken
 		}
 	}

pkg/server/options/options.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -29,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	genericapiserveroptions "k8s.io/apiserver/pkg/server/options"
 	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog/v2"
 	controlplaneapiserver "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 
 	etcdoptions "github.com/kcp-dev/embeddedetcd/options"
@@ -74,6 +76,9 @@ type ExtraOptions struct {
 	// --miniproxy-mapping-file flag of the front-proxy. Do NOT expose this flag to users via main server options.
 	// It is overridden by the kcp start command.
 	AdditionalMappingsFile string
+
+	// If CABundleFile is set, it contains the path to a file containing a PEM-encoded CA bundle to validate client certificates presented to kcp when using client-go internally.
+	CABundleFile string
 }
 
 type completedOptions struct {
@@ -114,8 +119,9 @@ func NewOptions(rootDir string) *Options {
 			DiscoveryPollInterval:              60 * time.Second,
 			ExperimentalBindFreePort:           false,
 			ConversionCELTransformationTimeout: time.Second,
+			CABundleFile:                       "",
 
-			BatteriesIncluded: sets.List[string](batteries.Defaults),
+			BatteriesIncluded: sets.List(batteries.Defaults),
 		},
 	}
 
@@ -168,6 +174,7 @@ func (o *Options) AddFlags(fss *cliflag.NamedFlagSets) {
 	fs.StringVar(&o.Extra.LogicalClusterAdminKubeconfig, "logical-cluster-admin-kubeconfig", o.Extra.LogicalClusterAdminKubeconfig, "Kubeconfig holding system:kcp:logical-cluster-admin credentials for connecting to other shards. Defaults to the loopback client")
 	fs.StringVar(&o.Extra.ExternalLogicalClusterAdminKubeconfig, "external-logical-cluster-admin-kubeconfig", o.Extra.ExternalLogicalClusterAdminKubeconfig, "Kubeconfig holding system:kcp:external-logical-cluster-admin credentials for connecting to the external address (e.g. the front-proxy). Defaults to the loopback client")
 	fs.StringVar(&o.Extra.RootIdentitiesFile, "root-identities-file", "", "Path to a YAML file used to bootstrap APIExport identities inside the root workspace. The YAML file must be structured as {\"identities\": [ {\"export\": \"<APIExport name>\", \"identity\": \"<APIExport identity>\"}, ... ]}. If a secret with matching APIExport name already exists inside kcp-system namespace, it will be left unchanged. Defaults to empty, i.e. no identities are bootstrapped.")
+	fs.StringVar(&o.Extra.CABundleFile, "ca-bundle-file", o.Extra.CABundleFile, "Path to a file containing a PEM-encoded CA bundle. If set, this CA bundle will be used to validate client certificates presented to the kcp when using client-go internally.")
 
 	fs.BoolVar(&o.Extra.ExperimentalBindFreePort, "experimental-bind-free-port", o.Extra.ExperimentalBindFreePort, "Bind to a free port. --secure-port must be 0. Use the admin.kubeconfig to extract the chosen port.")
 	fs.MarkHidden("experimental-bind-free-port") //nolint:errcheck
@@ -228,7 +235,7 @@ func (o *CompletedOptions) Validate() []error {
 		}
 	}
 
-	batterySet := sets.New[string](o.Extra.BatteriesIncluded...)
+	batterySet := sets.New(o.Extra.BatteriesIncluded...)
 	if batterySet.Has(batteries.User) && !batterySet.Has(batteries.Admin) {
 		errs = append(errs, fmt.Errorf("battery %s enabled which requires %s as well", batteries.User, batteries.Admin))
 	}
@@ -237,6 +244,20 @@ func (o *CompletedOptions) Validate() []error {
 		errs = append(errs, fmt.Errorf("--shard-external-url is required if --logical-cluster-admin-kubeconfig is set"))
 	}
 
+	if o.Extra.CABundleFile != "" {
+		data, err := os.ReadFile(o.Extra.CABundleFile)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("ca-bundle-file %q could not be read: %w", o.Extra.CABundleFile, err))
+		} else {
+			if len(data) == 0 {
+				errs = append(errs, fmt.Errorf("ca-bundle-file %q is empty", o.Extra.CABundleFile))
+			} else {
+				// TODO: validate if this is valid PEM data?
+				klog.Infof("ca-bundle-file %q read successfully, %d bytes", o.Extra.CABundleFile, len(data))
+			}
+		}
+	}
+
 	return errs
 }
 
@@ -374,15 +395,15 @@ func (o *Options) Complete(ctx context.Context, rootDir string) (*CompletedOptio
 		}
 	}
 	if differential {
-		bats := sets.New[string](sets.List[string](batteries.Defaults)...)
+		bats := sets.New(sets.List(batteries.Defaults)...)
 		for _, b := range o.Extra.BatteriesIncluded {
 			if strings.HasPrefix(b, "+") {
 				bats.Insert(b[1:])
 			} else if strings.HasPrefix(b, "-") {
 				bats.Delete(b[1:])
 			}
 		}
-		o.Extra.BatteriesIncluded = sets.List[string](bats)
+		o.Extra.BatteriesIncluded = sets.List(bats)
 	}
 
 	completedEmbeddedEtcd := o.EmbeddedEtcd.Complete(o.GenericControlPlane.Etcd)
@@ -399,6 +420,16 @@ func (o *Options) Complete(ctx context.Context, rootDir string) (*CompletedOptio
 		return nil, err
 	}
 
+	if o.Extra.CABundleFile != "" {
+		if !filepath.IsAbs(o.Extra.CABundleFile) {
+			pwd, err := os.Getwd()
+			if err != nil {
+				return nil, err
+			}
+			o.Extra.CABundleFile = filepath.Join(pwd, o.Extra.CABundleFile)
+		}
+	}
+
 	return &CompletedOptions{
 		completedOptions: &completedOptions{
 			GenericControlPlane: completedGenericOptions,