diff --git a/cmd/sharded-test-server/frontproxy.go b/cmd/sharded-test-server/frontproxy.go index a028715eb0f..eaeaca55efe 100644 --- a/cmd/sharded-test-server/frontproxy.go +++ b/cmd/sharded-test-server/frontproxy.go @@ -39,6 +39,7 @@ import ( "k8s.io/klog/v2" "github.com/kcp-dev/kcp/cmd/test-server/helpers" + "github.com/kcp-dev/kcp/pkg/server/proxy/types" kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" kcptestingserver "github.com/kcp-dev/kcp/sdk/testing/server" "github.com/kcp-dev/kcp/sdk/testing/third_party/library-go/crypto" @@ -66,15 +67,7 @@ func startFrontProxy( logger := klog.FromContext(ctx) - type mappingEntry struct { - Path string `json:"path"` - Backend string `json:"backend"` - BackendServerCA string `json:"backend_server_ca"` - ProxyClientCert string `json:"proxy_client_cert"` - ProxyClientKey string `json:"proxy_client_key"` - } - - mappings := []mappingEntry{ + mappings := []types.PathMapping{ { Path: "/services/", // TODO: support multiple virtual workspace backend servers @@ -83,9 +76,17 @@ func startFrontProxy( ProxyClientCert: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.crt"), ProxyClientKey: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.key"), }, + { + Path: "/e2e/clusters/{cluster}/", + Backend: "https://localhost:2443", + BackendServerCA: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"), + // in the existing testcases, these two do not matter, but have to be non-empty + ProxyClientCert: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.crt"), + ProxyClientKey: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.key"), + }, { Path: "/clusters/", - // TODO: support multiple shard backend servers + // this path is not actually used, since shard URLs are determined based on the Shard Backend: "https://localhost:6444", BackendServerCA: filepath.Join(workDirPath, ".kcp", "serving-ca.crt"), ProxyClientCert: filepath.Join(workDirPath, ".kcp-front-proxy", "requestheader.crt"), diff --git a/pkg/proxy/lookup/lookup.go b/pkg/proxy/lookup/lookup.go index d26e29433d1..82b9eea7bf7 100644 --- a/pkg/proxy/lookup/lookup.go +++ b/pkg/proxy/lookup/lookup.go @@ -30,63 +30,138 @@ import ( "github.com/kcp-dev/logicalcluster/v3" kcpauthorization "github.com/kcp-dev/kcp/pkg/authorization" + "github.com/kcp-dev/kcp/pkg/index" proxyindex "github.com/kcp-dev/kcp/pkg/proxy/index" + "github.com/kcp-dev/kcp/pkg/server/proxy/types" ) -func WithClusterResolver(delegate http.Handler, index proxyindex.Index) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - var cs = strings.SplitN(strings.TrimLeft(req.URL.Path, "/"), "/", 3) - if len(cs) < 2 || cs[0] != "clusters" { - delegate.ServeHTTP(w, req) - return +func WithClusterResolver(delegate http.Handler, mappings []types.PathMapping, index proxyindex.Index) http.Handler { + mux := http.NewServeMux() + + // fallback for all unrecognized URLs + mux.Handle("/", delegate) + + // Use the extra path mappings as an additional source of cluster names in URLs; + // it's okay for a virtual workspace URL to not match here or to not have a + // cluster placeholder in its URL pattern, since the default handler will simply + // forward the request unchanged (and most likely, unauthenticated). + + // We can use the same handler for all mappings, since the actual muxing to + // the destinations happens later in proxy.HttpHandler; here we only care about + // detecting the cluster name. + mappingHandler := newMappingHandler(delegate, index) + + for _, mapping := range mappings { + p := strings.TrimRight(mapping.Path, "/") + + // Even though we know how to handle the "special" core clusters path, + // the mapping provides additional PKI configuration that is not available + // by just looking up the cluster in the index and figuring out the + // target shard. That's why it's required to configure /clusters/ in the + // front-proxy mappings and since admins could choose not to include it, + // we only enable the built-in clusterResolveHandler if we actually find + // an appropriate mapping. + if p == "/clusters" { + // we know how to parse cluster URLs + resolveHandler := newClusterResolveHandler(delegate, index) + mux.HandleFunc("/clusters/{cluster}", resolveHandler) + mux.HandleFunc("/clusters/{cluster}/{trail...}", resolveHandler) + } else { + // mappings are configured with *prefixes*; in order to match both exact matches + // and prefix matches (i.e. if "/foo" is configured, both "/foo" and "/foo/bar" + // must match), each mapping is added twice to the mux. + mux.HandleFunc(p, mappingHandler) + mux.HandleFunc(p+"/{trail...}", mappingHandler) } + } - ctx := req.Context() - logger := klog.FromContext(ctx) - attributes, err := filters.GetAuthorizerAttributes(ctx) - if err != nil { - responsewriters.InternalError(w, req, err) - return - } + return mux +} - clusterPath := logicalcluster.NewPath(cs[1]) - if !clusterPath.IsValid() { - // this includes wildcards - logger.WithValues("requestPath", req.URL.Path).V(4).Info("Invalid cluster path") - responsewriters.Forbidden(req.Context(), attributes, w, req, kcpauthorization.WorkspaceAccessNotPermittedReason, kubernetesscheme.Codecs) - return - } +func newClusterResolveHandler(delegate http.Handler, index proxyindex.Index) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + clusterName := req.PathValue("cluster") - result, found := index.LookupURL(clusterPath) - if result.ErrorCode != 0 { - http.Error(w, "Not available.", result.ErrorCode) - return - } - if !found { - logger.WithValues("clusterPath", clusterPath).V(4).Info("Unknown cluster path") - responsewriters.Forbidden(req.Context(), attributes, w, req, kcpauthorization.WorkspaceAccessNotPermittedReason, kubernetesscheme.Codecs) + req, result := resolveClusterName(w, req, index, clusterName) + if req == nil { return } + shardURL, err := url.Parse(result.URL) if err != nil { responsewriters.InternalError(w, req, err) return } - logger.WithValues("from", "/clusters/"+cs[1], "to", shardURL).V(4).Info("Redirecting") + ctx := req.Context() + + logger := klog.FromContext(ctx) + logger.WithValues("from", "/clusters/"+clusterName, "to", shardURL).V(4).Info("Redirecting") + shardURL.RawQuery = req.URL.RawQuery shardURL.Path = strings.TrimSuffix(shardURL.Path, "/") - if len(cs) == 3 { - shardURL.Path += "/" + cs[2] + if trail := req.PathValue("trail"); len(trail) != 0 { + shardURL.Path += "/" + trail } ctx = WithShardURL(ctx, shardURL) - ctx = WithClusterName(ctx, result.Cluster) - ctx = WithWorkspaceType(ctx, result.Type) req = req.WithContext(ctx) delegate.ServeHTTP(w, req) - }) + } +} + +func newMappingHandler(delegate http.Handler, index proxyindex.Index) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + // not every virtual workspace and/or every mapping has a {cluster} in its URL; + // also wildcard requests have to be passed without lookup + clusterName := req.PathValue("cluster") + if clusterName == "" || clusterName == "*" { + delegate.ServeHTTP(w, req) + return + } + + req, _ = resolveClusterName(w, req, index, clusterName) + if req == nil { + return + } + + delegate.ServeHTTP(w, req) + } +} + +func resolveClusterName(w http.ResponseWriter, req *http.Request, index proxyindex.Index, clusterName string) (*http.Request, *index.Result) { + ctx := req.Context() + logger := klog.FromContext(ctx) + attributes, err := filters.GetAuthorizerAttributes(ctx) + if err != nil { + responsewriters.InternalError(w, req, err) + return nil, nil + } + + clusterPath := logicalcluster.NewPath(clusterName) + if !clusterPath.IsValid() { + // this includes wildcards + logger.WithValues("requestPath", req.URL.Path).V(4).Info("Invalid cluster path") + responsewriters.Forbidden(req.Context(), attributes, w, req, kcpauthorization.WorkspaceAccessNotPermittedReason, kubernetesscheme.Codecs) + return nil, nil + } + + result, found := index.LookupURL(clusterPath) + if result.ErrorCode != 0 { + http.Error(w, "Not available.", result.ErrorCode) + return nil, nil + } + if !found { + logger.WithValues("clusterPath", clusterPath).V(4).Info("Unknown cluster path") + responsewriters.Forbidden(req.Context(), attributes, w, req, kcpauthorization.WorkspaceAccessNotPermittedReason, kubernetesscheme.Codecs) + return nil, nil + } + + ctx = WithClusterName(ctx, result.Cluster) + ctx = WithWorkspaceType(ctx, result.Type) + + return req.WithContext(ctx), &result } type lookupKey int diff --git a/pkg/proxy/mapping.go b/pkg/proxy/mapping.go index af95f5566ca..b0380403f7d 100644 --- a/pkg/proxy/mapping.go +++ b/pkg/proxy/mapping.go @@ -29,17 +29,17 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog/v2" - "github.com/kcp-dev/kcp/pkg/proxy/index" "github.com/kcp-dev/kcp/pkg/server/proxy" + "github.com/kcp-dev/kcp/pkg/server/proxy/types" ) -func loadMappings(filename string) ([]proxy.PathMapping, error) { +func loadMappings(filename string) ([]types.PathMapping, error) { mappingData, err := os.ReadFile(filename) if err != nil { return nil, err } - var mapping []proxy.PathMapping + var mapping []types.PathMapping if err := yaml.Unmarshal(mappingData, &mapping); err != nil { return nil, err } @@ -47,14 +47,13 @@ func loadMappings(filename string) ([]proxy.PathMapping, error) { return mapping, nil } -func isShardMapping(m proxy.PathMapping) bool { +func isShardMapping(m types.PathMapping) bool { return m.Path == "/clusters/" } -func NewHandler(ctx context.Context, mappings []proxy.PathMapping, index index.Index) (http.Handler, error) { +func NewHandler(ctx context.Context, mappings []types.PathMapping) (http.Handler, error) { handlers := proxy.HttpHandler{ - Index: index, - Mappings: proxy.HttpHandlerMappings{ + Mappings: types.HttpHandlerMappings{ { Weight: 0, Path: "/metrics", @@ -108,7 +107,7 @@ func NewHandler(ctx context.Context, mappings []proxy.PathMapping, index index.I if m.Path == "/" { handlers.DefaultHandler = handler } else { - handlers.Mappings = append(handlers.Mappings, proxy.HttpHandlerMapping{ + handlers.Mappings = append(handlers.Mappings, types.HttpHandlerMapping{ Weight: len(m.Path), Path: m.Path, Handler: handler, diff --git a/pkg/proxy/server.go b/pkg/proxy/server.go index 8d960155847..89a9d8a1cfb 100644 --- a/pkg/proxy/server.go +++ b/pkg/proxy/server.go @@ -93,7 +93,7 @@ func NewServer(ctx context.Context, c CompletedConfig) (*Server, error) { // interface. s.IndexController = index.NewController(ctx, s.KcpSharedInformerFactory.Core().V1alpha1().Shards(), getClientFunc) - handler, err := NewHandler(ctx, mappings, s.IndexController) + handler, err := NewHandler(ctx, mappings) if err != nil { return s, err } @@ -131,7 +131,7 @@ func NewServer(ctx context.Context, c CompletedConfig) (*Server, error) { if hasShardMapping { // This middleware must happen before the authentication. - handler = lookup.WithClusterResolver(handler, s.IndexController) + handler = lookup.WithClusterResolver(handler, mappings, s.IndexController) } requestInfoFactory := requestinfo.NewFactory() diff --git a/pkg/server/localproxy.go b/pkg/server/localproxy.go index 2f894a571e1..2a497e0ba6f 100644 --- a/pkg/server/localproxy.go +++ b/pkg/server/localproxy.go @@ -42,6 +42,7 @@ import ( "github.com/kcp-dev/kcp/pkg/proxy/lookup" "github.com/kcp-dev/kcp/pkg/server/filters" "github.com/kcp-dev/kcp/pkg/server/proxy" + "github.com/kcp-dev/kcp/pkg/server/proxy/types" corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" corev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/core/v1alpha1" @@ -211,7 +212,7 @@ func WithLocalProxy( } // If additional mappings file is provided, read it and add the mappings to the handler - handlers, err := NewLocalProxyHandler(defaultHandlerFunc, indexState, additionalMappingsFile) + handlers, err := NewLocalProxyHandler(defaultHandlerFunc, additionalMappingsFile) if err != nil { return nil, fmt.Errorf("failed to create local proxy handler: %w", err) } @@ -223,8 +224,8 @@ func WithLocalProxy( // This function is very similar to proxy/mapping.go.NewHandler. // If we want to re-use that code, we basically would be merging proxy with server packages. // Which is not desirable at the point of writing (2024-10-26), but might be in the future. -func NewLocalProxyHandler(defaultHandler http.Handler, index index.Index, additionalMappingsFile string) (http.Handler, error) { - mapping := []proxy.PathMapping{} +func NewLocalProxyHandler(defaultHandler http.Handler, additionalMappingsFile string) (http.Handler, error) { + mapping := []types.PathMapping{} if additionalMappingsFile != "" { mappingData, err := os.ReadFile(additionalMappingsFile) if err != nil { @@ -237,8 +238,7 @@ func NewLocalProxyHandler(defaultHandler http.Handler, index index.Index, additi } handlers := proxy.HttpHandler{ - Index: index, - Mappings: proxy.HttpHandlerMappings{ + Mappings: types.HttpHandlerMappings{ { Weight: 0, Path: "/metrics", @@ -280,7 +280,7 @@ func NewLocalProxyHandler(defaultHandler http.Handler, index index.Index, additi handler = withProxyAuthHeaders(handler, userHeader, groupHeader, extraHeaderPrefix) - handlers.Mappings = append(handlers.Mappings, proxy.HttpHandlerMapping{ + handlers.Mappings = append(handlers.Mappings, types.HttpHandlerMapping{ Weight: len(m.Path), Path: m.Path, Handler: handler, diff --git a/pkg/server/proxy/handler.go b/pkg/server/proxy/handler.go index 52a3649d9bb..635778a92cf 100644 --- a/pkg/server/proxy/handler.go +++ b/pkg/server/proxy/handler.go @@ -18,111 +18,43 @@ package proxy import ( "net/http" - "net/url" - "path" "strings" - "github.com/kcp-dev/logicalcluster/v3" - - "github.com/kcp-dev/kcp/pkg/proxy/index" + "github.com/kcp-dev/kcp/pkg/proxy/lookup" + "github.com/kcp-dev/kcp/pkg/server/proxy/types" ) -// PathMapping describes how to route traffic from a path to a backend server. -// Each Path is registered with the DefaultServeMux with a handler that -// delegates to the specified backend. -type PathMapping struct { - Path string `json:"path"` - Backend string `json:"backend"` - BackendServerCA string `json:"backend_server_ca"` - ProxyClientCert string `json:"proxy_client_cert"` - ProxyClientKey string `json:"proxy_client_key"` - UserHeader string `json:"user_header,omitempty"` - GroupHeader string `json:"group_header,omitempty"` - ExtraHeaderPrefix string `json:"extra_header_prefix"` -} - type HttpHandler struct { - Index index.Index - Mappings HttpHandlerMappings + Mappings types.HttpHandlerMappings DefaultHandler http.Handler } -// httpHandlerMapping is used to route traffic to the correct backend server. -// Higher weight means that the mapping is more specific and should be matched first. -type HttpHandlerMapping struct { - Weight int - Path string - Handler http.Handler -} - -type HttpHandlerMappings []HttpHandlerMapping - -// Sort mappings by weight -// higher weight means that the mapping is more specific and should be matched first -// Example: /clusters/cluster1/ will be matched before /clusters/ . -func (h HttpHandlerMappings) Sort() { - for i := range h { - for j := range h { - if h[i].Weight > h[j].Weight { - h[i], h[j] = h[j], h[i] - } - } - } -} - func (h *HttpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - // mappings are used to route traffic to the correct backend server. - // It should not have `/clusters` as prefix because that is handled by the - // shardHandler or mounts. Logic is as follows: - // 1. We detect URL for the request and find the correct handler. URL can be - // shard based, virtual workspace or mount. First two are covered by r.URL, - // where mounts are covered by annotation on the workspace with the mount path. - // 2. If mountpoint is found, we rewrite the URL to resolve, else use one in - // request to match with mappings. - // 3. Iterate over mappings and find the one that matches the URL. If found, - // use the handler for that mapping, else use default handler - kcp. - // Mappings are done from most specific to least specific: - // Example: /clusters/cluster1/ will be matched before /clusters/ . - url, errorCode := h.resolveURL(r) - if errorCode != 0 { - http.Error(w, http.StatusText(errorCode), errorCode) - return - } - - for _, m := range h.Mappings { - if strings.HasPrefix(url, m.Path) { - m.Handler.ServeHTTP(w, r) - return - } + // we already parsed and looked up the cluster in the clusterresolver middleware, + // and potentially have stored the shard URL in the context already + shardURL := lookup.ShardURLFrom(r.Context()) + if shardURL != nil { + r.URL = shardURL } - h.DefaultHandler.ServeHTTP(w, r) -} + mux := http.NewServeMux() -func (h *HttpHandler) resolveURL(r *http.Request) (string, int) { - // if we don't match any of the paths, use the default behavior - request - var cs = strings.SplitN(strings.TrimLeft(r.URL.Path, "/"), "/", 3) - if len(cs) < 2 || cs[0] != "clusters" { - return r.URL.Path, 0 + // fallback for all unrecognized URLs + if h.DefaultHandler != nil { + mux.Handle("/", h.DefaultHandler) } - clusterPath := logicalcluster.NewPath(cs[1]) - if !clusterPath.IsValid() { - return r.URL.Path, 0 - } + for _, mapping := range h.Mappings { + p := strings.TrimRight(mapping.Path, "/") - result, found := h.Index.LookupURL(clusterPath) - if result.ErrorCode != 0 { - return "", result.ErrorCode - } - if found { - u, err := url.Parse(result.URL) - if err == nil && u != nil { - u.Path = strings.TrimSuffix(u.Path, "/") - r.URL.Path = path.Join(u.Path, strings.Join(cs[2:], "/")) // override request prefix and keep kube api contextual suffix - return u.Path, 0 + if p == "/clusters" { + mux.Handle("/clusters/{cluster}", mapping.Handler) + mux.Handle("/clusters/{cluster}/{trail...}", mapping.Handler) + } else { + mux.Handle(p, mapping.Handler) + mux.Handle(p+"/{trail...}", mapping.Handler) } } - return r.URL.Path, 0 + mux.ServeHTTP(w, r) } diff --git a/pkg/server/proxy/types/mapping.go b/pkg/server/proxy/types/mapping.go new file mode 100644 index 00000000000..bab7ee31283 --- /dev/null +++ b/pkg/server/proxy/types/mapping.go @@ -0,0 +1,56 @@ +/* +Copyright 2024 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import "net/http" + +// PathMapping describes how to route traffic from a path to a backend server. +// Each Path is registered with the DefaultServeMux with a handler that +// delegates to the specified backend. +type PathMapping struct { + Path string `json:"path"` + Backend string `json:"backend"` + BackendServerCA string `json:"backend_server_ca"` + ProxyClientCert string `json:"proxy_client_cert"` + ProxyClientKey string `json:"proxy_client_key"` + UserHeader string `json:"user_header,omitempty"` + GroupHeader string `json:"group_header,omitempty"` + ExtraHeaderPrefix string `json:"extra_header_prefix"` +} + +// httpHandlerMapping is used to route traffic to the correct backend server. +// Higher weight means that the mapping is more specific and should be matched first. +type HttpHandlerMapping struct { + Weight int + Path string + Handler http.Handler +} + +type HttpHandlerMappings []HttpHandlerMapping + +// Sort mappings by weight +// higher weight means that the mapping is more specific and should be matched first +// Example: /clusters/cluster1/ will be matched before /clusters/ . +func (h HttpHandlerMappings) Sort() { + for i := range h { + for j := range h { + if h[i].Weight > h[j].Weight { + h[i], h[j] = h[j], h[i] + } + } + } +} diff --git a/pkg/server/proxy/mapping_test.go b/pkg/server/proxy/types/mapping_test.go similarity index 98% rename from pkg/server/proxy/mapping_test.go rename to pkg/server/proxy/types/mapping_test.go index dea77ddb5c3..c87bec5f7f4 100644 --- a/pkg/server/proxy/mapping_test.go +++ b/pkg/server/proxy/types/mapping_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package proxy +package types import ( "reflect" diff --git a/test/e2e/authentication/workspace_test.go b/test/e2e/authentication/workspace_test.go index be588b71041..32a703f13ce 100644 --- a/test/e2e/authentication/workspace_test.go +++ b/test/e2e/authentication/workspace_test.go @@ -58,7 +58,7 @@ func TestWorkspaceOIDC(t *testing.T) { kcpClusterClient, err := kcpclientset.NewForConfig(kcpConfig) require.NoError(t, err) - // start a two mock OIDC servers that will listen on random ports + // start two mock OIDC servers that will listen on random ports // (only for discovery and keyset handling, no actual login workflows) mockA, ca := authfixtures.StartMockOIDC(t, server) mockB, _ := authfixtures.StartMockOIDC(t, server) diff --git a/test/e2e/proxy/proxy_test.go b/test/e2e/proxy/proxy_test.go new file mode 100644 index 00000000000..76aa25d1917 --- /dev/null +++ b/test/e2e/proxy/proxy_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package proxy + +import ( + "context" + "net" + "net/http" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" + "github.com/kcp-dev/logicalcluster/v3" + + tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" + kcptesting "github.com/kcp-dev/kcp/sdk/testing" + "github.com/kcp-dev/kcp/test/e2e/fixtures/authfixtures" + "github.com/kcp-dev/kcp/test/e2e/framework" +) + +func TestMappingWithClusterContext(t *testing.T) { + framework.Suite(t, "control-plane") + + ctx := context.Background() + + // start kcp and setup clients; + // note that the sharded-test-server will configure a special + // `/e2e/clusters/{cluster}` route, which we are testing here + server := kcptesting.SharedKcpServer(t) + + if len(server.ShardNames()) < 2 { + t.Skip("This test requires a multi-shard setup with front-proxy.") + } + + // The goal is to prove that having a {cluster} placeholder in the URL + // correctly provides a cluster context to the underlying handler. This + // handler is normally a custom virtual workspace, since kcp itself can + // only handle health endpoints and /clusters/ URLs. + // Instead of wiring up a custom virtual workspace, we just start a minimal + // echo server to see if the correct headers are received. + var lastHeaders http.Header + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + lastHeaders = r.Header.Clone() + }) + + srv := &http.Server{ + Handler: handler, + BaseContext: func(l net.Listener) context.Context { + return t.Context() + }, + Addr: "localhost:2443", // as per front-proxy mapping + } + + dirPath := filepath.Dir(server.KubeconfigPath()) + go func() { + if err := srv.ListenAndServeTLS( + filepath.Join(dirPath, "apiserver.crt"), + filepath.Join(dirPath, "apiserver.key"), + ); err != nil { + t.Logf("Error: %v", err) + } + }() + + baseWsPath, _ := kcptesting.NewWorkspaceFixture(t, server, logicalcluster.NewPath("root"), kcptesting.WithNamePrefix("oidc")) + + kcpConfig := server.BaseConfig(t) + kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(kcpConfig) + require.NoError(t, err) + kcpClusterClient, err := kcpclientset.NewForConfig(kcpConfig) + require.NoError(t, err) + + // start a mock OIDC servers that will listen on a random port + // (only for discovery and keyset handling, no actual login workflows) + mock, ca := authfixtures.StartMockOIDC(t, server) + + // setup a new workspace auth config that uses mockoidc's server + authConfig := authfixtures.CreateWorkspaceOIDCAuthentication(t, ctx, kcpClusterClient, baseWsPath, mock, ca, nil) + + // use these configs in new WorkspaceTypes + wsType := authfixtures.CreateWorkspaceType(t, ctx, kcpClusterClient, baseWsPath, "with-oidc", authConfig) + + // create a new workspace with our new type + t.Log("Creating Workspace...") + teamPath, teamWs := kcptesting.NewWorkspaceFixture(t, server, baseWsPath, kcptesting.WithName("team-a"), kcptesting.WithType(baseWsPath, tenancyv1alpha1.WorkspaceTypeName(wsType))) + teamCluster := teamWs.Spec.Cluster + + // grant permissions to the user + authfixtures.GrantWorkspaceAccess(t, ctx, kubeClusterClient, teamPath, "grant-oidc-user", "cluster-admin", []rbacv1.Subject{{ + Kind: "User", + Name: "oidc:user@example.com", + }, { + Kind: "Group", + Name: "oidc:developers", + }}) + + var ( + username = "billybob" + email = "bob@example.com" + groups = []string{"developers"} + + expectedScope = "cluster:" + teamCluster + expectedClusterName = teamCluster + expectedUsername = "oidc:" + email + expectedGroups = []string{"system:authenticated"} + ) + + for _, group := range groups { + expectedGroups = append(expectedGroups, "oidc:"+group) + } + + token := authfixtures.CreateOIDCToken(t, mock, username, email, groups) + + cfg := framework.ConfigWithToken(token, kcpConfig) + cfg.Host += "/e2e" + + client, err := kcpkubernetesclientset.NewForConfig(cfg) + require.NoError(t, err) + + require.Eventually(t, func() bool { + _, _ = client.Cluster(teamPath).CoreV1().ConfigMaps("default").List(ctx, metav1.ListOptions{}) + + if lastHeaders == nil { + return false + } + + // if we find anything else in the auth infos, no need to try any further, neither the token + // nor the OIDC mapping will change to produce other values + require.Equal(t, expectedScope, lastHeaders.Get("X-Remote-Extra-Authentication.kcp.io%2fscopes")) + require.Equal(t, expectedClusterName, lastHeaders.Get("X-Remote-Extra-Authentication.kcp.io%2fcluster-Name")) + require.Equal(t, expectedUsername, lastHeaders.Get("X-Remote-User")) + require.ElementsMatch(t, expectedGroups, lastHeaders.Values("X-Remote-Group")) + + return true + }, wait.ForeverTestTimeout, 500*time.Millisecond) +}