From 102ae780b25e785ae3052e29105cf12f592cfb31 Mon Sep 17 00:00:00 2001 From: Robert Vasek Date: Thu, 2 Oct 2025 17:05:04 +0200 Subject: [PATCH 1/7] e2e: add CRD helper to wildwest fixture On-behalf-of: @SAP robert.vasek@sap.com Signed-off-by: Robert Vasek --- test/e2e/fixtures/wildwest/bootstrap.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/e2e/fixtures/wildwest/bootstrap.go b/test/e2e/fixtures/wildwest/bootstrap.go index 223f5bfa3f2..68fdd87a2b3 100644 --- a/test/e2e/fixtures/wildwest/bootstrap.go +++ b/test/e2e/fixtures/wildwest/bootstrap.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apiextensionsv1client "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -55,3 +56,14 @@ func FakePClusterCreate(t *testing.T, client apiextensionsv1client.CustomResourc err := configcrds.CreateFromFS(ctx, client, rawCustomResourceDefinitions, grs...) require.NoError(t, err) } + +// CRD returns an *apiextensionsv1.CustomResourceDefinition for the GroupResource specified by gr from +// rawCustomResourceDefinitions. The embedded file's name must have the format _.yaml. +func CRD(t *testing.T, gr metav1.GroupResource) *apiextensionsv1.CustomResourceDefinition { + t.Helper() + + crd, err := configcrds.CRD(rawCustomResourceDefinitions, gr) + require.NoError(t, err, "error decoding CRD") + + return crd +} From 57f2ef691c7a6e5ea527f06ac530fd60ca4b6d88 Mon Sep 17 00:00:00 2001 From: Robert Vasek Date: Thu, 2 Oct 2025 17:28:57 +0200 Subject: [PATCH 2/7] reverting 'Add CachedResourceSchemaSource for CachedResource #3553' CachedResourceSchemaSource is not needed anymore. APIExport holds the reference to the schema for virtual resources. On-behalf-of: @SAP robert.vasek@sap.com Signed-off-by: Robert Vasek --- .../cachedresources_controller.go | 155 +--- .../cachedresources_reconcile.go | 17 +- ...dresources_reconcile_schema_replication.go | 279 ------ ...urces_reconcile_schema_replication_test.go | 817 ------------------ .../cachedresources_reconcile_schemasource.go | 239 ----- ...edresources_reconcile_schemasource_test.go | 538 ------------ pkg/server/controllers.go | 27 +- .../cache/v1alpha1/types_cachedresource.go | 61 +- 8 files changed, 16 insertions(+), 2117 deletions(-) delete mode 100644 pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication.go delete mode 100644 pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication_test.go delete mode 100644 pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource.go delete mode 100644 pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource_test.go diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_controller.go b/pkg/reconciler/cache/cachedresources/cachedresources_controller.go index 46660185796..18190cc4f5b 100644 --- a/pkg/reconciler/cache/cachedresources/cachedresources_controller.go +++ b/pkg/reconciler/cache/cachedresources/cachedresources_controller.go @@ -23,11 +23,8 @@ import ( "time" corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -36,34 +33,21 @@ import ( "k8s.io/klog/v2" kcpcache "github.com/kcp-dev/apimachinery/v2/pkg/cache" - kcpapiextensionsclientset "github.com/kcp-dev/client-go/apiextensions/client" - kcpapiextensionsv1informers "github.com/kcp-dev/client-go/apiextensions/informers/apiextensions/v1" kcpdynamic "github.com/kcp-dev/client-go/dynamic" kcpcorev1informers "github.com/kcp-dev/client-go/informers/core/v1" kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" "github.com/kcp-dev/logicalcluster/v3" - cacheclient "github.com/kcp-dev/kcp/pkg/cache/client" - "github.com/kcp-dev/kcp/pkg/cache/client/shard" - "github.com/kcp-dev/kcp/pkg/indexers" "github.com/kcp-dev/kcp/pkg/informer" "github.com/kcp-dev/kcp/pkg/logging" replicationcontroller "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresources/replication" "github.com/kcp-dev/kcp/pkg/reconciler/committer" "github.com/kcp-dev/kcp/pkg/reconciler/dynamicrestmapper" - "github.com/kcp-dev/kcp/pkg/tombstone" - apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" - apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" cachev1alpha1client "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/typed/cache/v1alpha1" kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" - apisv1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha1" - apisv1alpha2informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha2" cacheinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/cache/v1alpha1" - corev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/core/v1alpha1" - apisv1alpha1listers "github.com/kcp-dev/kcp/sdk/client/listers/apis/v1alpha1" cachev1alpha1listers "github.com/kcp-dev/kcp/sdk/client/listers/cache/v1alpha1" ) @@ -83,14 +67,12 @@ func NewController( shardName string, kcpClusterClient kcpclientset.ClusterInterface, kcpCacheClient kcpclientset.ClusterInterface, - crdClusterClient kcpapiextensionsclientset.ClusterInterface, dynamicClient kcpdynamic.ClusterInterface, cacheDynamicClient kcpdynamic.ClusterInterface, kubeClusterClient kcpkubernetesclientset.ClusterInterface, namespaceInformer kcpcorev1informers.NamespaceClusterInformer, secretInformer kcpcorev1informers.SecretClusterInformer, - crdInformer kcpapiextensionsv1informers.CustomResourceDefinitionClusterInformer, dynRESTMapper *dynamicrestmapper.DynamicRESTMapper, @@ -99,15 +81,6 @@ func NewController( cachedResourceInformer cacheinformers.CachedResourceClusterInformer, cachedResourceEndpointSliceInformer cacheinformers.CachedResourceEndpointSliceClusterInformer, - - logicalClusterInformer corev1alpha1informers.LogicalClusterClusterInformer, - apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer, - - apiExportInformer apisv1alpha2informers.APIExportClusterInformer, - globalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer, - - apiResourceSchemaInformer apisv1alpha1informers.APIResourceSchemaClusterInformer, - globalAPIResourceSchemaInformer apisv1alpha1informers.APIResourceSchemaClusterInformer, ) (*Controller, error) { c := &Controller{ shardName: shardName, @@ -171,105 +144,13 @@ func NewController( return err }, - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return logicalClusterInformer.Cluster(cluster).Lister().Get(corev1alpha1.LogicalClusterName) - }, - getAPIBinding: func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) { - return apiBindingInformer.Cluster(cluster).Lister().Get(name) - }, - getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { - return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport](apisv1alpha2.Resource("apiexports"), apiExportInformer.Informer().GetIndexer(), globalAPIExportInformer.Informer().GetIndexer(), path, name) - }, - getAPIResourceSchema: informer.NewScopedGetterWithFallback[*apisv1alpha1.APIResourceSchema, apisv1alpha1listers.APIResourceSchemaLister](apiResourceSchemaInformer.Lister(), globalAPIResourceSchemaInformer.Lister()), - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return apiResourceSchemaInformer.Cluster(cluster).Lister().Get(name) - }, - - listCRDsByGR: func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) { - crds, err := crdInformer.Cluster(cluster).Lister().List(labels.Everything()) - if err != nil { - return nil, err - } - - var crdsWithGR []*apiextensionsv1.CustomResourceDefinition - for _, crd := range crds { - if crd.Spec.Group == gr.Group && crd.Spec.Names.Plural == gr.Resource { - crdsWithGR = append(crdsWithGR, crd) - } - } - return crdsWithGR, nil - }, - - getCRD: func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { - crd, err := crdInformer.Lister().Cluster(cluster).Get(name) - if err == nil { - return crd, nil - } - - // In case the lister is slow to catch up, try a live read - crd, err = crdClusterClient.Cluster(cluster.Path()).ApiextensionsV1().CustomResourceDefinitions().Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return nil, err - } - - return crd, nil - }, - - createCachedAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - _, err := kcpClusterClient.Cluster(cluster.Path()).ApisV1alpha1().APIResourceSchemas().Create(ctx, sch, metav1.CreateOptions{}) - return err - }, - - updateCreateAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - _, err := kcpClusterClient.Cluster(cluster.Path()).ApisV1alpha1().APIResourceSchemas().Update(ctx, sch, metav1.UpdateOptions{}) - return err - }, - controllerRegistry: newRegistry(), } _, _ = cachedResourceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { c.enqueueCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), "") }, - UpdateFunc: func(_, obj interface{}) { - c.enqueueCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), "") - }, - DeleteFunc: func(obj interface{}) { c.enqueueCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), "") }, - }) - - _, _ = logicalClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of LogicalCluster update") - }, - UpdateFunc: func(_, obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of LogicalCluster update") - }, - DeleteFunc: func(obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of LogicalCluster update") - }, - }) - - _, _ = crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of CRD update") - }, - UpdateFunc: func(_, obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of CRD update") - }, - DeleteFunc: func(obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of CRD update") - }, - }) - - _, _ = apiResourceSchemaInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of APIResourceSchema update") - }, - UpdateFunc: func(_, obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of APIResourceSchema update") - }, - DeleteFunc: func(obj interface{}) { - c.enqueueCachedResourcesInCluster(obj.(metav1.Object), " because of APIResourceSchema update") - }, + AddFunc: func(obj interface{}) { c.enqueue(obj) }, + UpdateFunc: func(_, obj interface{}) { c.enqueue(obj) }, + DeleteFunc: func(obj interface{}) { c.enqueue(obj) }, }) return c, nil @@ -310,42 +191,19 @@ type Controller struct { getEndpointSlice func(ctx context.Context, clusterName logicalcluster.Name, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) createEndpointSlice func(ctx context.Context, clusterName logicalcluster.Path, endpointSlice *cachev1alpha1.CachedResourceEndpointSlice) error - getLogicalCluster func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) - getAPIBinding func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) - getAPIExport func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) - getAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) - getLocalAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) - listCRDsByGR func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) - - getCRD func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) - - createCachedAPIResourceSchema func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error - updateCreateAPIResourceSchema func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error - controllerRegistry *controllerRegistry started bool } -func (c *Controller) enqueueCachedResourcesInCluster(metaObj metav1.Object, logSuffix string) { - cachedResources, err := c.CachedResourceLister.Cluster(logicalcluster.From(metaObj)).List(labels.Everything()) - if err != nil { - utilruntime.HandleError(err) - return - } - for _, cr := range cachedResources { - c.enqueueCachedResource(cr, logSuffix) - } -} - -func (c *Controller) enqueueCachedResource(cachedResource *cachev1alpha1.CachedResource, logSuffix string) { - key, err := kcpcache.MetaClusterNamespaceKeyFunc(cachedResource) +func (c *Controller) enqueue(obj interface{}) { + key, err := kcpcache.MetaClusterNamespaceKeyFunc(obj) if err != nil { utilruntime.HandleError(err) return } logger := logging.WithQueueKey(logging.WithReconciler(klog.Background(), ControllerName), key) - logger.V(4).Info(fmt.Sprintf("queueing CachedResource%s", logSuffix)) + logger.V(4).Info("queueing CachedResource") c.queue.Add(key) } @@ -355,7 +213,6 @@ func (c *Controller) Start(ctx context.Context, numThreads int) { defer c.queue.ShutDown() logger := logging.WithReconciler(klog.FromContext(ctx), ControllerName) - ctx = klog.NewContext(cacheclient.WithShardInContext(ctx, shard.New(c.shardName)), logger) ctx = klog.NewContext(ctx, logger) logger.Info("Starting controller") defer logger.Info("Shutting down controller") diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile.go b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile.go index 07aed50ce99..c1f6398c837 100644 --- a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile.go +++ b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile.go @@ -36,7 +36,6 @@ import ( "github.com/kcp-dev/kcp/pkg/crypto" "github.com/kcp-dev/kcp/pkg/logging" replicationcontroller "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresources/replication" - "github.com/kcp-dev/kcp/pkg/tombstone" apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" ) @@ -65,20 +64,6 @@ func (c *Controller) reconcile(ctx context.Context, cluster logicalcluster.Name, createIdentitySecret: c.createIdentitySecret, secretNamespace: c.secretNamespace, }, - &schemaSource{ - getLogicalCluster: c.getLogicalCluster, - getAPIBinding: c.getAPIBinding, - getAPIExport: c.getAPIExport, - getAPIResourceSchema: c.getAPIResourceSchema, - listCRDsByGR: c.listCRDsByGR, - }, - &replicateResourceSchema{ - getAPIResourceSchema: c.getAPIResourceSchema, - getLocalAPIResourceSchema: c.getLocalAPIResourceSchema, - getCRD: c.getCRD, - createCachedAPIResourceSchema: c.createCachedAPIResourceSchema, - updateCreateAPIResourceSchema: c.updateCreateAPIResourceSchema, - }, &endpointSlice{ getEndpointSlice: c.getEndpointSlice, createEndpointSlice: c.createEndpointSlice, @@ -103,7 +88,7 @@ func (c *Controller) reconcile(ctx context.Context, cluster logicalcluster.Name, dynRESTMapper: c.dynRESTMapper, cacheKcpInformers: c.cacheKcpInformers, discoveringDynamicKcpInformers: c.discoveringDynamicKcpInformers, - callback: func(obj interface{}) { c.enqueueCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), "") }, + callback: c.enqueue, controllerRegistry: c.controllerRegistry, }, } diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication.go b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication.go deleted file mode 100644 index 18e3a3e06b9..00000000000 --- a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication.go +++ /dev/null @@ -1,279 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cachedresources - -import ( - "context" - "fmt" - - apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/klog/v2" - - "github.com/kcp-dev/logicalcluster/v3" - - apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" - cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" - "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" -) - -type replicateResourceSchema struct { - getAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) - getLocalAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) - getCRD func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) - - createCachedAPIResourceSchema func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error - updateCreateAPIResourceSchema func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error -} - -func CachedAPIResourceSchemaName(cachedResourceUID types.UID, gr schema.GroupResource) string { - return fmt.Sprintf("cachedresources-cache-kcp-io-%s.%s", cachedResourceUID, gr.String()) -} - -func ownAPIResourceSchema(cr *cachev1alpha1.CachedResource, sch *apisv1alpha1.APIResourceSchema) { - sch.Name = CachedAPIResourceSchemaName(cr.UID, schema.GroupResource{ - Group: cr.Spec.Group, - Resource: cr.Spec.Resource, - }) - sch.OwnerReferences = []metav1.OwnerReference{ - { - APIVersion: cachev1alpha1.SchemeGroupVersion.String(), - Kind: "CachedResource", - Name: cr.Name, - UID: cr.UID, - }, - } -} - -func (r *replicateResourceSchema) reconcile(ctx context.Context, cachedResource *cachev1alpha1.CachedResource) (reconcileStatus, error) { - if !cachedResource.DeletionTimestamp.IsZero() { - return reconcileStatusContinue, nil - } - if conditions.IsFalse(cachedResource, cachev1alpha1.CachedResourceSchemaSourceValid) { - return reconcileStatusStopAndRequeue, nil - } - logger := klog.FromContext(ctx) - - gvr := schema.GroupVersionResource{ - Group: cachedResource.Spec.Group, - Version: cachedResource.Spec.Version, - Resource: cachedResource.Spec.Resource, - } - - // We need to find out if we have the source schema replicated. - - // Get only the local schema. Global informer may be lagging behind - // and return the schema even if it's been already deleted locally. - _, err := r.getLocalAPIResourceSchema(logicalcluster.From(cachedResource), CachedAPIResourceSchemaName(cachedResource.UID, gvr.GroupResource())) - replicatedSchemaNotFound := apierrors.IsNotFound(err) - if err != nil && !replicatedSchemaNotFound { - logger.Error(err, "failed to get replicated APIResourceSchema") - return reconcileStatusStopAndRequeue, err - } - - if cachedResource.Status.ResourceSchemaSource.APIResourceSchema != nil { - if replicatedSchemaNotFound { - // We don't, so it needs to be created. - - sourceSchema, err := r.getAPIResourceSchema(logicalcluster.Name(cachedResource.Status.ResourceSchemaSource.APIResourceSchema.ClusterName), cachedResource.Status.ResourceSchemaSource.APIResourceSchema.Name) - if err != nil { - logger.Error(err, "failed to get source APIResourceSchema") - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - "Failed to get source APIResourceSchema: %v.", - err, - ) - return reconcileStatusStopAndRequeue, err - } - - if !validateAPIResourceSchemaForGVR(sourceSchema, gvr) { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "Schema is not valid. Please contact the APIExport owner to resolve.", - ) - return reconcileStatusStop, nil - } - - sch := sourceSchema.DeepCopy() - sch.ResourceVersion = "" // This is a copy, we need to reset the RV. - ownAPIResourceSchema(cachedResource, sch) - - if err = r.createCachedAPIResourceSchema(ctx, logicalcluster.From(cachedResource), sch); err != nil { - logger.Error(err, "failed to create the cached APIResourceSchema") - if !apierrors.IsAlreadyExists(err) { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - "Failed to store schema: %v.", - err, - ) - return reconcileStatusStopAndRequeue, err - } - } - - conditions.MarkTrue(cachedResource, cachev1alpha1.CachedResourceSourceSchemaReplicated) - return reconcileStatusStopAndRequeue, nil - } - - // The replicated APIResoureSchema already exists. - // No need to check for updates because it is immutable. - return reconcileStatusContinue, nil - } - - if cachedResource.Status.ResourceSchemaSource.CRD != nil { - crd, err := r.getCRD(ctx, logicalcluster.From(cachedResource), cachedResource.Status.ResourceSchemaSource.CRD.Name) - if err != nil { - logger.Error(err, "failed to get source CRD") - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - "Failed to get source CRD: %v.", - err, - ) - return reconcileStatusStopAndRequeue, err - } - - if apiextensionshelpers.IsCRDConditionFalse(crd, apiextensionsv1.Established) { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "API not ready.", - ) - return reconcileStatusStop, nil - } - - if !validateCRDForGVR(crd, gvr) { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "CRD %s does not define the requested group, resource or version.", - crd.Name, - ) - return reconcileStatusStop, nil - } - - if replicatedSchemaNotFound || crd.ObjectMeta.ResourceVersion != cachedResource.Status.ResourceSchemaSource.CRD.ResourceVersion { - // Either the replicated schema doesn't exist, or it needs updating. - - sourceSchema, err := apisv1alpha1.CRDToAPIResourceSchema(crd, "prefix") - if err != nil { - logger.Error(err, "failed to convert CRD to APIResourceSchema") - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - "Internal error while processing source CRD.", - ) - return reconcileStatusStopAndRequeue, err - } - ownAPIResourceSchema(cachedResource, sourceSchema) - - if replicatedSchemaNotFound { - if err = r.createCachedAPIResourceSchema(ctx, logicalcluster.From(cachedResource), sourceSchema); err != nil { - logger.Error(err, "failed to replicate APIResourceSchema") - if !apierrors.IsAlreadyExists(err) { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - "Failed to replicate schema: %v.", - err, - ) - return reconcileStatusStopAndRequeue, err - } - } - } else { - if err = r.updateCreateAPIResourceSchema(ctx, logicalcluster.From(cachedResource), sourceSchema); err != nil { - logger.Error(err, "failed to update the replicated APIResourceSchema") - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - "Failed to update the replicated schema: %v.", - err, - ) - return reconcileStatusStopAndRequeue, err - } - } - - conditions.MarkTrue(cachedResource, cachev1alpha1.CachedResourceSourceSchemaReplicated) - cachedResource.Status.ResourceSchemaSource.CRD.ResourceVersion = crd.ObjectMeta.ResourceVersion - return reconcileStatusStopAndRequeue, nil - } - - return reconcileStatusContinue, nil - } - - // This should never happen! - - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "API not ready.", - ) - cachedResource.Status.ResourceSchemaSource = nil - - return reconcileStatusStopAndRequeue, nil -} - -func validateAPIResourceSchemaForGVR(sch *apisv1alpha1.APIResourceSchema, gvr schema.GroupVersionResource) bool { - if sch.Spec.Group != gvr.Group || sch.Spec.Names.Plural != gvr.Resource { - return false - } - for i := range sch.Spec.Versions { - if sch.Spec.Versions[i].Name == gvr.Version { - return true - } - } - return false -} - -func validateCRDForGVR(crd *apiextensionsv1.CustomResourceDefinition, gvr schema.GroupVersionResource) bool { - if crd.Spec.Group != gvr.Group || crd.Spec.Names.Plural != gvr.Resource { - return false - } - for _, version := range crd.Status.StoredVersions { - if version == gvr.Version { - return true - } - } - return false -} diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication_test.go b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication_test.go deleted file mode 100644 index c0e0a392d60..00000000000 --- a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schema_replication_test.go +++ /dev/null @@ -1,817 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cachedresources - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/require" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" - - "github.com/kcp-dev/logicalcluster/v3" - - apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" - cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" - "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" -) - -func TestReconcileReplicateResourceSchema(t *testing.T) { - tests := map[string]struct { - CachedResource *cachev1alpha1.CachedResource - reconciler *replicateResourceSchema - expectedErr error - expectedStatus reconcileStatus - expectedConditions conditionsv1alpha1.Conditions - }{ - // - // Common - // - - "has deletion timestamp and should skip": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: ptr.To(metav1.Now()), - }, - }, - reconciler: &replicateResourceSchema{}, - expectedStatus: reconcileStatusContinue, - }, - "has CachedResourceSchemaSourceValid=false condition, and should abort": { - CachedResource: &cachev1alpha1.CachedResource{ - Status: cachev1alpha1.CachedResourceStatus{ - Conditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "", - ), - }, - }, - }, - reconciler: &replicateResourceSchema{}, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "", - ), - }, - }, - - // - // APIResourceSchemaSource - // - - "APIResourceSchemaSource with replicated schema": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{}, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return &apisv1alpha1.APIResourceSchema{}, nil - }, - }, - expectedStatus: reconcileStatusContinue, - }, - "APIResourceSchemaSource with missing replicated schema and missing source schema": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{ - ClusterName: "providers_cowboys_cluster_name", - Name: "missing-cowboys-schema", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - `Failed to get source APIResourceSchema: apiresourceschemas.apis.kcp.io "missing-cowboys-schema" not found.`, - ), - }, - expectedErr: fmt.Errorf(`apiresourceschemas.apis.kcp.io "missing-cowboys-schema" not found`), - }, - "APIResourceSchemaSource with missing replicated schema and invalid source schema": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{ - ClusterName: "providers_cowboys_cluster_name", - Name: "today.cowboys.wildwest.dev", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "calmwest.org", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowgirls", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1", - }, - }, - }, - }, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - }, - expectedStatus: reconcileStatusStop, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - `Schema is not valid. Please contact the APIExport owner to resolve.`, - ), - }, - }, - "APIResourceSchemaSource with missing replicated schema and failing create": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{ - ClusterName: "providers_cowboys_cluster_name", - Name: "today.cowboys.wildwest.dev", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - createCachedAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - return fmt.Errorf("create failed") - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - `Failed to store schema: create failed.`, - ), - }, - expectedErr: fmt.Errorf("create failed"), - }, - "APIResourceSchemaSource with missing replicated schema succeeds": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{ - ClusterName: "providers_cowboys_cluster_name", - Name: "today.cowboys.wildwest.dev", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - createCachedAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - return nil - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition(cachev1alpha1.CachedResourceSourceSchemaReplicated), - }, - expectedErr: nil, - }, - - // - // CRDSchemaSource - // - - "CRDSchemaSource with up-to-date replicated schema": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getCRD: func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - StoredVersions: []string{"v1alpha1"}, - }, - }, nil - }, - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - "consumer_cluster_name": { - "cachedresources-cache-kcp-io-cowboys-cr-uid.cowboys.wildwest.dev": &apisv1alpha1.APIResourceSchema{}, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - "consumer_cluster_name": { - "cachedresources-cache-kcp-io-cowboys-cr-uid.cowboys.wildwest.dev": &apisv1alpha1.APIResourceSchema{}, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - }, - expectedStatus: reconcileStatusContinue, - }, - "CRDSchemaSource with missing replicated schema fails": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getCRD: func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - StoredVersions: []string{"v1alpha1"}, - }, - }, nil - }, - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - createCachedAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - return fmt.Errorf("create failed") - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - `Failed to replicate schema: create failed.`, - ), - }, - expectedErr: fmt.Errorf("create failed"), - }, - "CRDSchemaSource with missing replicated schema succeeds": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getCRD: func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - StoredVersions: []string{"v1alpha1"}, - }, - }, nil - }, - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - createCachedAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - return nil - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition(cachev1alpha1.CachedResourceSourceSchemaReplicated), - }, - }, - "CRDSchemaSource with out-of-date replicated schema fails": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - ResourceVersion: "old", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getCRD: func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - StoredVersions: []string{"v1alpha1"}, - }, - }, nil - }, - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - "consumer_cluster_name": { - "cachedresources-cache-kcp-io-cowboys-cr-uid.cowboys.wildwest.dev": &apisv1alpha1.APIResourceSchema{}, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - updateCreateAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - return fmt.Errorf("update failed") - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSourceSchemaReplicated, - cachev1alpha1.SourceSchemaReplicatedFailedReason, - conditionsv1alpha1.ConditionSeverityError, - `Failed to update the replicated schema: update failed.`, - ), - }, - expectedErr: fmt.Errorf("update failed"), - }, - "CRDSchemaSource with out-of-date replicated schema succeeds": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-cr", - UID: "cowboys-cr-uid", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "consumer_cluster_name", - }, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - Status: cachev1alpha1.CachedResourceStatus{ - ResourceSchemaSource: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - ResourceVersion: "old", - }, - }, - }, - }, - reconciler: &replicateResourceSchema{ - getCRD: func(ctx context.Context, cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { - return &apiextensionsv1.CustomResourceDefinition{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - ResourceVersion: "latest", - }, - Spec: apiextensionsv1.CustomResourceDefinitionSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - StoredVersions: []string{"v1alpha1"}, - }, - }, nil - }, - getLocalAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - m := map[logicalcluster.Name]map[string]*apisv1alpha1.APIResourceSchema{ - "providers_cowboys_cluster_name": { - "today.cowboys.wildwest.dev": { - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Group: "wildwest.dev", - Names: apiextensionsv1.CustomResourceDefinitionNames{ - Plural: "cowboys", - }, - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, - }, - "consumer_cluster_name": { - "cachedresources-cache-kcp-io-cowboys-cr-uid.cowboys.wildwest.dev": &apisv1alpha1.APIResourceSchema{}, - }, - } - if sch := m[cluster][name]; sch != nil { - return sch, nil - } - return nil, apierrors.NewNotFound(apisv1alpha1.Resource("apiresourceschemas"), name) - }, - updateCreateAPIResourceSchema: func(ctx context.Context, cluster logicalcluster.Name, sch *apisv1alpha1.APIResourceSchema) error { - return nil - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition(cachev1alpha1.CachedResourceSourceSchemaReplicated), - }, - }, - } - - for testName, tt := range tests { - t.Run(testName, func(t *testing.T) { - status, err := tt.reconciler.reconcile(context.Background(), tt.CachedResource) - - resetLastTransitionTime(tt.expectedConditions) - resetLastTransitionTime(tt.CachedResource.Status.Conditions) - - if tt.expectedErr != nil { - require.Error(t, err) - require.Equal(t, tt.expectedErr.Error(), err.Error()) - } else { - require.NoError(t, err) - } - - require.Equal(t, tt.expectedStatus, status, "reconcile status mismatch") - require.Equal(t, tt.expectedConditions, tt.CachedResource.Status.Conditions, "conditions mismatch") - }) - } -} diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource.go b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource.go deleted file mode 100644 index 30f2f61cf16..00000000000 --- a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource.go +++ /dev/null @@ -1,239 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cachedresources - -import ( - "context" - "fmt" - - apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/klog/v2" - - "github.com/kcp-dev/logicalcluster/v3" - - "github.com/kcp-dev/kcp/pkg/reconciler/apis/apibinding" - apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" - apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" - cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" - conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" - "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" -) - -type schemaSource struct { - getLogicalCluster func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) - getAPIBinding func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) - getAPIExport func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) - getAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) - listCRDsByGR func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) -} - -func (r *schemaSource) reconcile(ctx context.Context, cachedResource *cachev1alpha1.CachedResource) (reconcileStatus, error) { - logger := klog.FromContext(ctx) - - if !cachedResource.DeletionTimestamp.IsZero() { - return reconcileStatusContinue, nil - } - if conditions.IsTrue(cachedResource, cachev1alpha1.CachedResourceSchemaSourceValid) { - return reconcileStatusContinue, nil - } - - cachedResource.Status.ResourceSchemaSource = nil - conditions.Delete(cachedResource, cachev1alpha1.CachedResourceSourceSchemaReplicated) - - gvr := schema.GroupVersionResource{ - Group: cachedResource.Spec.Group, - Version: cachedResource.Spec.Version, - Resource: cachedResource.Spec.Resource, - } - cluster := logicalcluster.From(cachedResource) - - // Find out where the schema comes from. - - // Start with inspecting the LogicalCluster to find out if we have an associated - // APIBinding for this GR, which could mean the schema comes from an APIResourceSchema. - - lc, err := r.getLogicalCluster(cluster) - if err != nil { - return reconcileStatusStopAndRequeue, err - } - - boundResources, err := apibinding.GetResourceBindings(lc) - if err != nil { - return reconcileStatusStopAndRequeue, err - } - - notReadyCond := conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "API not ready.", - ) - - if bindingLock, found := boundResources[gvr.GroupResource().String()]; found { - if bindingLock.Name != "" { - // This resource's schema originates from an APIResourceSchema - // because we have an associated APIBinding. - - apiBinding, err := r.getAPIBinding(cluster, bindingLock.Name) - if err != nil { - return reconcileStatusStopAndRequeue, err - } - - apiExport, err := r.getAPIExport(logicalcluster.NewPath(apiBinding.Spec.Reference.Export.Path), apiBinding.Spec.Reference.Export.Name) - if err != nil { - return reconcileStatusStopAndRequeue, err - } - - var schemaName string - for _, res := range apiExport.Spec.Resources { - if res.Group != gvr.Group || res.Name != gvr.Resource { - continue - } - if res.Storage.CRD == nil { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "Schema %s in APIExport %s:%s is incompatible. Please contact the APIExport owner to resolve.", - res.Schema, - apiBinding.Spec.Reference.Export.Path, - apiBinding.Spec.Reference.Export.Name, - ) - return reconcileStatusStop, nil - } - schemaName = res.Schema - break - } - - if schemaName == "" { - // The LogicalCluster is holding a lock for a GR that is - // not defined in the associated APIExport??? - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "No valid schema available in APIExport %s:%s. Please contact the APIExport owner to resolve.", - apiBinding.Spec.Reference.Export.Path, - apiBinding.Spec.Reference.Export.Name, - ) - return reconcileStatusStop, nil - } - - sourceSchema, err := r.getAPIResourceSchema(logicalcluster.From(apiExport), schemaName) - if err != nil { - return reconcileStatusStopAndRequeue, err - } - - var hasRequestedVersion bool - for i := range sourceSchema.Spec.Versions { - if sourceSchema.Spec.Versions[i].Name == gvr.Version { - hasRequestedVersion = true - break - } - } - - if !hasRequestedVersion { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "Schema %s in APIExport %s:%s does not define the requested resource version. Please contact the APIExport owner to resolve.", - schemaName, - apiBinding.Spec.Reference.Export.Path, - apiBinding.Spec.Reference.Export.Name, - ) - return reconcileStatusStop, nil - } - - conditions.MarkTrue(cachedResource, cachev1alpha1.CachedResourceSchemaSourceValid) - cachedResource.Status.ResourceSchemaSource = &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{ - ClusterName: logicalcluster.From(sourceSchema).String(), - Name: sourceSchema.Name, - }, - } - return reconcileStatusContinue, nil - } else if bindingLock.CRD { - // The resource is backed by a CRD. Fall through to find that CRD. - } else { - // This should never happen! Neither APIBinding or CRD are present in the binding lock. - // We can drop this item from the queue. We'll try again once the LogicalCluster annotation is updated. - - logger.Error(nil, "failed to process bindings annotation on LogicalCluster", - "LogicalCluster", fmt.Sprintf("%s|%s", cluster, corev1alpha1.LogicalClusterName), - "annotationKey", apibinding.ResourceBindingsAnnotationKey, - "annotation", lc.Annotations[apibinding.ResourceBindingsAnnotationKey]) - conditions.Set(cachedResource, notReadyCond) - return reconcileStatusStop, nil - } - } - - // Maybe it's a CRD? - - crds, err := r.listCRDsByGR(cluster, gvr.GroupResource()) - if err != nil { - return reconcileStatusStopAndRequeue, err - } - - if len(crds) != 1 { - // Zero means there is nothing serving this GR at the moment, and >1 is impossible. - conditions.Set(cachedResource, notReadyCond) - return reconcileStatusStop, nil - } - - // It's definitely a CRD! - - crd := crds[0] - - if apiextensionshelpers.IsCRDConditionFalse(crd, apiextensionsv1.Established) { - conditions.Set(cachedResource, notReadyCond) - return reconcileStatusStop, nil - } - - var hasRequestedVersion bool - for _, version := range crd.Status.StoredVersions { - if version == gvr.Version { - hasRequestedVersion = true - break - } - } - if !hasRequestedVersion { - conditions.MarkFalse( - cachedResource, - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "CRD %s does not define the requested resource version.", - crd.Name, - ) - return reconcileStatusStop, nil - } - - conditions.MarkTrue(cachedResource, cachev1alpha1.CachedResourceSchemaSourceValid) - cachedResource.Status.ResourceSchemaSource = &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: crd.Name, - }, - } - return reconcileStatusStopAndRequeue, nil -} diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource_test.go b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource_test.go deleted file mode 100644 index 648fbeb435e..00000000000 --- a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_schemasource_test.go +++ /dev/null @@ -1,538 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package cachedresources - -import ( - "context" - "testing" - - "github.com/stretchr/testify/require" - - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/utils/ptr" - - "github.com/kcp-dev/logicalcluster/v3" - - "github.com/kcp-dev/kcp/pkg/reconciler/apis/apibinding" - apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" - apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" - cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" - conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" - "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" -) - -func TestReconcileSchemaSource(t *testing.T) { - tests := map[string]struct { - CachedResource *cachev1alpha1.CachedResource - reconciler *schemaSource - expectedErr error - expectedStatus reconcileStatus - expectedConditions conditionsv1alpha1.Conditions - expectedSchemaSrc *cachev1alpha1.CachedResourceSchemaSource - }{ - // - // Common - // - - "has deletion timestamp and should skip": { - CachedResource: &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - DeletionTimestamp: ptr.To(metav1.Now()), - }, - }, - reconciler: &schemaSource{}, - expectedStatus: reconcileStatusContinue, - }, - "has CachedResourceSchemaSourceValid=true condition and should skip": { - CachedResource: &cachev1alpha1.CachedResource{ - Status: cachev1alpha1.CachedResourceStatus{ - Conditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - ), - }, - }, - }, - reconciler: &schemaSource{}, - expectedStatus: reconcileStatusContinue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - ), - }, - }, - "bad internal.apis.kcp.io/resource-bindings annotation on LogicalCluster": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - apibinding.ResourceBindingsAnnotationKey: `{"cowboys.wildwest.dev": {}}`, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStop, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "API not ready.", - ), - }, - expectedSchemaSrc: nil, - }, - - // - // APIResourceSchemaSource - // - - "APIResourceSchemaSource with missing resource": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - apibinding.ResourceBindingsAnnotationKey: `{"cowboys.wildwest.dev": {"n": "cowboys-binding"}}`, - }, - }, - }, nil - }, - getAPIBinding: func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) { - return &apisv1alpha2.APIBinding{ - Spec: apisv1alpha2.APIBindingSpec{ - Reference: apisv1alpha2.BindingReference{ - Export: &apisv1alpha2.ExportBindingReference{ - Path: "providers:cowboys", - Name: "cowboys-export", - }, - }, - }, - }, nil - }, - getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { - return &apisv1alpha2.APIExport{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "providers_cowboys_cluster_name", - }, - }, - Spec: apisv1alpha2.APIExportSpec{ - Resources: []apisv1alpha2.ResourceSchema{}, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStop, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "No valid schema available in APIExport providers:cowboys:cowboys-export. Please contact the APIExport owner to resolve.", - ), - }, - expectedSchemaSrc: nil, - }, - "APIResourceSchemaSource with non-CRD storage": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - apibinding.ResourceBindingsAnnotationKey: `{"cowboys.wildwest.dev": {"n": "cowboys-binding"}}`, - }, - }, - }, nil - }, - getAPIBinding: func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) { - return &apisv1alpha2.APIBinding{ - Spec: apisv1alpha2.APIBindingSpec{ - Reference: apisv1alpha2.BindingReference{ - Export: &apisv1alpha2.ExportBindingReference{ - Path: "providers:cowboys", - Name: "cowboys-export", - }, - }, - }, - }, nil - }, - getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { - return &apisv1alpha2.APIExport{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "providers_cowboys_cluster_name", - }, - }, - Spec: apisv1alpha2.APIExportSpec{ - Resources: []apisv1alpha2.ResourceSchema{ - { - Group: "wildwest.dev", - Name: "cowboys", - Schema: "today.cowboys.wildwest.dev", - Storage: apisv1alpha2.ResourceSchemaStorage{}, - }, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStop, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "Schema today.cowboys.wildwest.dev in APIExport providers:cowboys:cowboys-export is incompatible. Please contact the APIExport owner to resolve.", - ), - }, - expectedSchemaSrc: nil, - }, - "APIResourceSchemaSource": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - apibinding.ResourceBindingsAnnotationKey: `{"cowboys.wildwest.dev": {"n": "cowboys-binding"}}`, - }, - }, - }, nil - }, - getAPIBinding: func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) { - return &apisv1alpha2.APIBinding{ - Spec: apisv1alpha2.APIBindingSpec{ - Reference: apisv1alpha2.BindingReference{ - Export: &apisv1alpha2.ExportBindingReference{ - Path: "providers:cowboys", - Name: "cowboys-export", - }, - }, - }, - }, nil - }, - getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { - return &apisv1alpha2.APIExport{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "providers_cowboys_cluster_name", - }, - }, - Spec: apisv1alpha2.APIExportSpec{ - Resources: []apisv1alpha2.ResourceSchema{ - { - Group: "wildwest.dev", - Name: "cowboys", - Schema: "today.cowboys.wildwest.dev", - Storage: apisv1alpha2.ResourceSchemaStorage{ - CRD: &apisv1alpha2.ResourceSchemaStorageCRD{}, - }, - }, - }, - }, - }, nil - }, - getAPIResourceSchema: func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return &apisv1alpha1.APIResourceSchema{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "providers_cowboys_cluster_name", - }, - Name: "today.cowboys.wildwest.dev", - }, - Spec: apisv1alpha1.APIResourceSchemaSpec{ - Versions: []apisv1alpha1.APIResourceVersion{ - { - Name: "v1alpha1", - }, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusContinue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - ), - }, - expectedSchemaSrc: &cachev1alpha1.CachedResourceSchemaSource{ - APIResourceSchema: &cachev1alpha1.APIResourceSchemaSource{ - ClusterName: "providers_cowboys_cluster_name", - Name: "today.cowboys.wildwest.dev", - }, - }, - }, - - // - // CRDSchemaSource - // - - "CRDSchemaSource but version is missing": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{}, nil - }, - listCRDsByGR: func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) { - return []*apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - StoredVersions: []string{"v1alpha2"}, - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStop, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceInvalidReason, - conditionsv1alpha1.ConditionSeverityError, - "CRD cowboys-crd does not define the requested resource version.", - ), - }, - expectedSchemaSrc: nil, - }, - "CRDSchemaSource but CRD is not Established": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{}, nil - }, - listCRDsByGR: func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) { - return []*apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - StoredVersions: []string{"v1alpha1"}, - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionFalse, - }, - }, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStop, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.FalseCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - cachev1alpha1.SchemaSourceNotReadyReason, - conditionsv1alpha1.ConditionSeverityError, - "API not ready.", - ), - }, - expectedSchemaSrc: nil, - }, - "CRDSchemaSource": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{}, nil - }, - listCRDsByGR: func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) { - return []*apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - StoredVersions: []string{"v1alpha1"}, - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - ), - }, - expectedSchemaSrc: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - }, - }, - }, - "CRDSchemaSource with claimed CRD": { - CachedResource: &cachev1alpha1.CachedResource{ - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: "wildwest.dev", - Version: "v1alpha1", - Resource: "cowboys", - }, - }, - }, - reconciler: &schemaSource{ - getLogicalCluster: func(cluster logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return &corev1alpha1.LogicalCluster{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - apibinding.ResourceBindingsAnnotationKey: `{"cowboys.wildwest.dev": {"c": true}}`, - }, - }, - }, nil - }, - listCRDsByGR: func(cluster logicalcluster.Name, gr schema.GroupResource) ([]*apiextensionsv1.CustomResourceDefinition, error) { - return []*apiextensionsv1.CustomResourceDefinition{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys-crd", - }, - Status: apiextensionsv1.CustomResourceDefinitionStatus{ - StoredVersions: []string{"v1alpha1"}, - Conditions: []apiextensionsv1.CustomResourceDefinitionCondition{ - { - Type: apiextensionsv1.Established, - Status: apiextensionsv1.ConditionTrue, - }, - }, - }, - }, - }, nil - }, - }, - expectedStatus: reconcileStatusStopAndRequeue, - expectedConditions: conditionsv1alpha1.Conditions{ - *conditions.TrueCondition( - cachev1alpha1.CachedResourceSchemaSourceValid, - ), - }, - expectedSchemaSrc: &cachev1alpha1.CachedResourceSchemaSource{ - CRD: &cachev1alpha1.CRDSchemaSource{ - Name: "cowboys-crd", - }, - }, - }, - } - - for testName, tt := range tests { - t.Run(testName, func(t *testing.T) { - status, err := tt.reconciler.reconcile(context.Background(), tt.CachedResource) - - resetLastTransitionTime(tt.expectedConditions) - resetLastTransitionTime(tt.CachedResource.Status.Conditions) - - if tt.expectedErr != nil { - require.Error(t, err) - require.Equal(t, tt.expectedErr.Error(), err.Error()) - } else { - require.NoError(t, err) - } - - require.Equal(t, tt.expectedStatus, status, "reconcile status mismatch") - require.Equal(t, tt.expectedConditions, tt.CachedResource.Status.Conditions, "conditions mismatch") - require.Equal(t, tt.expectedSchemaSrc, tt.CachedResource.Status.ResourceSchemaSource, "ResourceSchemaSource mismatch") - }) - } -} - -func resetLastTransitionTime(conditions conditionsv1alpha1.Conditions) { - // We don't care about LastTransitionTime. - for i := range conditions { - conditions[i].LastTransitionTime = metav1.Time{} - } -} diff --git a/pkg/server/controllers.go b/pkg/server/controllers.go index 1163e059b6a..2c8c34e81a4 100644 --- a/pkg/server/controllers.go +++ b/pkg/server/controllers.go @@ -1733,41 +1733,23 @@ func (s *Server) installCacheController(ctx context.Context, config *rest.Config if err != nil { return err } - crdClusterClient, err := kcpapiextensionsclientset.NewForConfig(workspaceConfig) - if err != nil { - return err - } cachedResourceInformer := s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResources() cachedResourceEndpointSliceInformer := s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices() - logicalClusterInformer := s.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters() - apiBindingInformer := s.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings() - apiExportInformer := s.KcpSharedInformerFactory.Apis().V1alpha2().APIExports() - globalAPIExportInformer := s.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports() - apiResourceSchemaInformer := s.KcpSharedInformerFactory.Apis().V1alpha1().APIResourceSchemas() - globalAPIResourceSchemaInformer := s.CacheKcpSharedInformerFactory.Apis().V1alpha1().APIResourceSchemas() c, err := cachedresources.NewController( s.Options.Extra.ShardName, kcpClusterClient, s.KcpCacheClusterClient, - crdClusterClient, dynamicClient, s.CacheDynamicClient, s.KubeClusterClient, s.KubeSharedInformerFactory.Core().V1().Namespaces(), s.KubeSharedInformerFactory.Core().V1().Secrets(), - s.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(), s.DynRESTMapper, s.DiscoveringDynamicSharedInformerFactory, s.CacheKcpSharedInformerFactory, cachedResourceInformer, cachedResourceEndpointSliceInformer, - logicalClusterInformer, - apiBindingInformer, - apiExportInformer, - globalAPIExportInformer, - apiResourceSchemaInformer, - globalAPIResourceSchemaInformer, ) if err != nil { return err @@ -1776,14 +1758,7 @@ func (s *Server) installCacheController(ctx context.Context, config *rest.Config Name: cachedresources.ControllerName, Wait: func(ctx context.Context, s *Server) error { return wait.PollUntilContextCancel(ctx, waitPollInterval, true, func(ctx context.Context) (bool, error) { - return cachedResourceInformer.Informer().HasSynced() && - cachedResourceEndpointSliceInformer.Informer().HasSynced() && - logicalClusterInformer.Informer().HasSynced() && - apiBindingInformer.Informer().HasSynced() && - apiExportInformer.Informer().HasSynced() && - globalAPIExportInformer.Informer().HasSynced() && - apiResourceSchemaInformer.Informer().HasSynced() && - globalAPIResourceSchemaInformer.Informer().HasSynced(), nil + return cachedResourceInformer.Informer().HasSynced() && cachedResourceEndpointSliceInformer.Informer().HasSynced(), nil }) }, Runner: func(ctx context.Context) { diff --git a/sdk/apis/cache/v1alpha1/types_cachedresource.go b/sdk/apis/cache/v1alpha1/types_cachedresource.go index 6c63a93e16c..833e49c6e47 100644 --- a/sdk/apis/cache/v1alpha1/types_cachedresource.go +++ b/sdk/apis/cache/v1alpha1/types_cachedresource.go @@ -133,17 +133,14 @@ const ( ) const ( - // CachedResourceSchemaSourceValid represents status of the schema reference. - CachedResourceSchemaSourceValid conditionsv1alpha1.ConditionType = "CachedResourceSchemaSourceValid" - - SchemaSourceNotReadyReason = "SchemaSourceNotReady" - SchemaSourceInvalidReason = "SchemaSourceInvalid" -) - -const ( - CachedResourceSourceSchemaReplicated conditionsv1alpha1.ConditionType = "CachedResourceSourceSchemaReplicated" - - SourceSchemaReplicatedFailedReason = "SourceSchemaReplicatedFailed" + // CachedResourceInvalidReferenceReason is a reason for the CachedResourceValid condition that the referenced + // CachedResource reference is invalid. + CachedResourceInvalidReferenceReason = "CachedResourceInvalidReference" + // CachedResourceNotFoundReason is a reason for the CachedResourceValid condition that the referenced CachedResource is not found. + CachedResourceNotFoundReason = "CachedResourceNotFound" + + // InternalErrorReason is a reason used by multiple conditions that something went wrong. + InternalErrorReason = "InternalError" ) // These are valid reasons of published resource. @@ -163,10 +160,6 @@ type CachedResourceStatus struct { // +optional ResourceCounts *ResourceCount `json:"resourceCounts,omitempty"` - // ResourceSchemaSource is a reference to the schema object of the cached resource. - // +optional - ResourceSchemaSource *CachedResourceSchemaSource `json:"resourceSchemaSource,omitempty"` - // Phase of the workspace (Initializing, Ready, Unavailable). // // +kubebuilder:default=Initializing @@ -177,44 +170,6 @@ type CachedResourceStatus struct { Conditions conditionsv1alpha1.Conditions `json:"conditions,omitempty"` } -// CachedResourceSchemaSource describes the source of resource schema. -// Exactly one field is set. -type CachedResourceSchemaSource struct { - // APIResourceSchema defines an APIResourceSchema as the source of the schema. - // +optional - APIResourceSchema *APIResourceSchemaSource `json:"apiResourceSchema,omitempty"` - - // CRD defines a CRD as the source of the schema. - // +optional - CRD *CRDSchemaSource `json:"crd,omitempty"` -} - -type APIResourceSchemaSource struct { - // ClusterName is the name of the cluster where the APIResourceSchema is defined. - // - // +required - // +kubebuilder:validation:MinLength=1 - ClusterName string `json:"clusterName"` - - // Name is the APIResourceSchema name. - // +required - // +kubebuilder:validation:MinLength=1 - Name string `json:"name"` -} - -type CRDSchemaSource struct { - // Name is the CRD name. - // - // +required - // +kubebuilder:validation:MinLength=1 - Name string `json:"name"` - - // ResourceVersion is the resource version of the source CRD object. - // - // +optional - ResourceVersion string `json:"resourceVersion"` -} - // ResourceCount is the number of resources that match the label selector // and are cached in the cache. type ResourceCount struct { From 281292e8e30861f047f16977a11d0c59f5c55905 Mon Sep 17 00:00:00 2001 From: Robert Vasek Date: Thu, 2 Oct 2025 17:51:54 +0200 Subject: [PATCH 3/7] CachedResources: fix index function name IndexByGVRAndShardAndLogicalClusterAndNamespace On-behalf-of: @SAP robert.vasek@sap.com Signed-off-by: Robert Vasek --- pkg/reconciler/cache/cachedresources/replication/indexers.go | 4 ++-- .../cachedresources/replication/replication_controller.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/reconciler/cache/cachedresources/replication/indexers.go b/pkg/reconciler/cache/cachedresources/replication/indexers.go index 0255af1900c..d11aa914406 100644 --- a/pkg/reconciler/cache/cachedresources/replication/indexers.go +++ b/pkg/reconciler/cache/cachedresources/replication/indexers.go @@ -34,8 +34,8 @@ const ( ByGVRAndLogicalClusterAndNamespace = "kcp-byGVRAndLogicalClusterAndNamespace" ) -// IndexByShardAndLogicalClusterAndNamespace is an index function that indexes by an object's shard and logical cluster, namespace and name. -func IndexByShardAndLogicalClusterAndNamespace(obj interface{}) ([]string, error) { +// IndexByGVRAndShardAndLogicalClusterAndNamespace is an index function that indexes by an object's shard and logical cluster, namespace and name. +func IndexByGVRAndShardAndLogicalClusterAndNamespace(obj interface{}) ([]string, error) { a, err := meta.Accessor(obj) if err != nil { return nil, err diff --git a/pkg/reconciler/cache/cachedresources/replication/replication_controller.go b/pkg/reconciler/cache/cachedresources/replication/replication_controller.go index 97f03d1a743..f8ee62379f1 100644 --- a/pkg/reconciler/cache/cachedresources/replication/replication_controller.go +++ b/pkg/reconciler/cache/cachedresources/replication/replication_controller.go @@ -238,7 +238,7 @@ func InstallIndexers(replicated *ReplicatedGVR) { indexers.AddIfNotPresentOrDie( replicated.Global.GetIndexer(), cache.Indexers{ - ByGVRAndShardAndLogicalClusterAndNamespaceAndName: IndexByShardAndLogicalClusterAndNamespace, + ByGVRAndShardAndLogicalClusterAndNamespaceAndName: IndexByGVRAndShardAndLogicalClusterAndNamespace, }, ) } From b6b5f35ed9a8459a9dccccb5f3956247ee645a67 Mon Sep 17 00:00:00 2001 From: Robert Vasek Date: Thu, 2 Oct 2025 17:55:17 +0200 Subject: [PATCH 4/7] Add virtual resources support Virtual resources add a way to project resources from a provider to consumer clusters using APIExports and APIBindings. On-behalf-of: @SAP robert.vasek@sap.com Signed-off-by: Robert Vasek --- config/crds/apis.kcp.io_apiexports.yaml-patch | 1 + pkg/authorization/bootstrap/policy.go | 8 + pkg/cache/server/bootstrap/bootstrap.go | 1 + pkg/endpointslice/endpointslice.go | 76 ++ pkg/indexers/apibinding.go | 25 + pkg/indexers/apiexport.go | 49 ++ .../apis/apibinding/apibinding_reconcile.go | 18 +- .../cachedresourceendpointslice_controller.go | 155 ++-- .../cachedresourceendpointslice_indexers.go | 43 +- .../cachedresourceendpointslice_reconcile.go | 196 +++-- ...hedresourceendpointslice_reconcile_test.go | 341 ++++----- ...hedresourceendpointsliceurls_controller.go | 369 ++++++++++ ...hedresourceendpointsliceurls_reconciler.go | 176 +++++ ...cachedresources_reconcile_identity_test.go | 7 + .../cachedresources/replication/indexers.go | 7 +- .../replication/replication_controller.go | 11 + .../aggregatingcrdversiondiscovery/config.go | 99 +++ .../aggregatingcrdversiondiscovery/server.go | 301 ++++++++ .../aggregatingcrdversiondiscovery/verbs.go | 170 +++++ pkg/server/apiextensions.go | 126 +++- pkg/server/config.go | 122 +++- pkg/server/controllers.go | 78 +- pkg/server/indexes.go | 24 +- pkg/server/server.go | 35 +- pkg/server/virtualresources/config.go | 113 +++ pkg/server/virtualresources/server.go | 395 +++++++++++ .../framework/virtualresource/context/keys.go | 38 + .../replication/authorizer/authorizer.go | 83 --- pkg/virtual/replication/authorizer/content.go | 224 ++++++ .../replication/authorizer/content_test.go | 668 ++++++++++++++++++ pkg/virtual/replication/builder/build.go | 267 +++++-- pkg/virtual/replication/builder/build_test.go | 55 +- pkg/virtual/replication/builder/unwrap.go | 206 +++++- sdk/apis/apis/v1alpha1/types_apibinding.go | 3 + sdk/apis/apis/v1alpha2/types_apiexport.go | 30 +- .../types_apiexport_conversion_test.go | 27 + .../types_cachedresourceendpointslice.go | 41 ++ .../vr_cachedresources_test.go | 590 ++++++++++++++++ 38 files changed, 4496 insertions(+), 682 deletions(-) create mode 100644 pkg/endpointslice/endpointslice.go create mode 100644 pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_controller.go create mode 100644 pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_reconciler.go create mode 100644 pkg/server/aggregatingcrdversiondiscovery/config.go create mode 100644 pkg/server/aggregatingcrdversiondiscovery/server.go create mode 100644 pkg/server/aggregatingcrdversiondiscovery/verbs.go create mode 100644 pkg/server/virtualresources/config.go create mode 100644 pkg/server/virtualresources/server.go create mode 100644 pkg/virtual/framework/virtualresource/context/keys.go delete mode 100644 pkg/virtual/replication/authorizer/authorizer.go create mode 100644 pkg/virtual/replication/authorizer/content.go create mode 100644 pkg/virtual/replication/authorizer/content_test.go create mode 100644 test/e2e/virtualresources/cachedresources/vr_cachedresources_test.go diff --git a/config/crds/apis.kcp.io_apiexports.yaml-patch b/config/crds/apis.kcp.io_apiexports.yaml-patch index 74d3549bc64..7a38e204310 100644 --- a/config/crds/apis.kcp.io_apiexports.yaml-patch +++ b/config/crds/apis.kcp.io_apiexports.yaml-patch @@ -17,6 +17,7 @@ path: /spec/versions/name=v1alpha2/schema/openAPIV3Schema/properties/spec/properties/resources/items/properties/storage/oneOf value: - required: ["crd"] + - required: ["virtual"] # conversion for core resources does not happen via webhooks, but is short-circuited to the # schema's Convert functions directly, but the CRD still needs to define a conversion. diff --git a/pkg/authorization/bootstrap/policy.go b/pkg/authorization/bootstrap/policy.go index ee234eee7ad..856c84911e2 100644 --- a/pkg/authorization/bootstrap/policy.go +++ b/pkg/authorization/bootstrap/policy.go @@ -25,6 +25,7 @@ import ( "k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy" "github.com/kcp-dev/kcp/sdk/apis/apis" + "github.com/kcp-dev/kcp/sdk/apis/cache" "github.com/kcp-dev/kcp/sdk/apis/core" "github.com/kcp-dev/kcp/sdk/apis/tenancy" ) @@ -109,6 +110,13 @@ func clusterRoles() []rbacv1.ClusterRole { rbacv1helpers.NewRule("get", "list", "watch").Groups(apis.GroupName).Resources("apiexportendpointslices").RuleOrDie(), }, }, + { + ObjectMeta: metav1.ObjectMeta{Name: SystemExternalLogicalClusterAdmin}, + Rules: []rbacv1.PolicyRule{ + rbacv1helpers.NewRule("update", "patch", "get").Groups(cache.GroupName).Resources("cachedresourceendpointslices/status").RuleOrDie(), + rbacv1helpers.NewRule("get", "list", "watch").Groups(cache.GroupName).Resources("cachedresourceendpointslices").RuleOrDie(), + }, + }, } } diff --git a/pkg/cache/server/bootstrap/bootstrap.go b/pkg/cache/server/bootstrap/bootstrap.go index 5fad961a4d3..012bfcfe20d 100644 --- a/pkg/cache/server/bootstrap/bootstrap.go +++ b/pkg/cache/server/bootstrap/bootstrap.go @@ -53,6 +53,7 @@ func Bootstrap(ctx context.Context, apiExtensionsClusterClient kcpapiextensionsc {"core.kcp.io", "shards"}, {"cache.kcp.io", "cachedobjects"}, {"cache.kcp.io", "cachedresources"}, + {"cache.kcp.io", "cachedresourceendpointslices"}, {"tenancy.kcp.io", "workspacetypes"}, {"rbac.authorization.k8s.io", "roles"}, {"rbac.authorization.k8s.io", "clusterroles"}, diff --git a/pkg/endpointslice/endpointslice.go b/pkg/endpointslice/endpointslice.go new file mode 100644 index 00000000000..f870d2387a9 --- /dev/null +++ b/pkg/endpointslice/endpointslice.go @@ -0,0 +1,76 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpointslice + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ListURLsFromUnstructured retrieves list of endpoint URLs from an unstructured object. +// The URLs are expected to be present at `.status.endpoints[].url` path inside the object. +func ListURLsFromUnstructured(endpointSlice unstructured.Unstructured) ([]string, error) { + endpoints, found, err := unstructured.NestedSlice(endpointSlice.Object, "status", "endpoints") + if err != nil { + return nil, fmt.Errorf("failed to get status.endpoints: %w", err) + } + if !found { + return nil, fmt.Errorf("status.endpoints not found") + } + + urls := make([]string, 0, len(endpoints)) + for i, ep := range endpoints { + endpointMap, ok := ep.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("endpoint at index %d is not an object", i) + } + + url, found, err := unstructured.NestedString(endpointMap, "url") + if err != nil { + return nil, fmt.Errorf("failed to get url from endpoint at index %d: %w", i, err) + } + if !found { + return nil, fmt.Errorf("missing url in endpoint at index %d", i) + } + + urls = append(urls, url) + } + + return urls, nil +} + +// FindOneURL finds exactly one URL with matching prefix in the urls slice. +// Multiple matches result in an error. +func FindOneURL(prefix string, urls []string) (string, error) { + var matches []string + for _, url := range urls { + if strings.HasPrefix(url, prefix) { + matches = append(matches, url) + } + } + + switch len(matches) { + case 1: + return matches[0], nil + case 0: + return "", fmt.Errorf("no URLs match prefix %q", prefix) + default: + return "", fmt.Errorf("ambiguous URLs %v with prefix %q", matches, prefix) + } +} diff --git a/pkg/indexers/apibinding.go b/pkg/indexers/apibinding.go index 012e0654406..78f3e112a11 100644 --- a/pkg/indexers/apibinding.go +++ b/pkg/indexers/apibinding.go @@ -26,6 +26,31 @@ import ( apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" ) +const ( + APIBindingByIdentityAndGroupResource = "apibinding-byIdentityGroupResource" +) + +// IndexAPIBindingByIdentityGroupResource is an index function that indexes an APIBinding by its +// bound resources' identity and group resource. +func IndexAPIBindingByIdentityGroupResource(obj interface{}) ([]string, error) { + apiBinding, ok := obj.(*apisv1alpha2.APIBinding) + if !ok { + return []string{}, fmt.Errorf("obj is supposed to be an APIBinding, but is %T", obj) + } + + ret := make([]string, 0, len(apiBinding.Status.BoundResources)) + + for _, r := range apiBinding.Status.BoundResources { + ret = append(ret, IdentityGroupResourceKeyFunc(r.Schema.IdentityHash, r.Group, r.Resource)) + } + + return ret, nil +} + +func IdentityGroupResourceKeyFunc(identity, group, resource string) string { + return fmt.Sprintf("%s/%s/%s", identity, group, resource) +} + // ClusterAndGroupResourceValue returns the index value for use with // IndexAPIBindingByClusterAndAcceptedClaimedGroupResources from clusterName and groupResource. func ClusterAndGroupResourceValue(clusterName logicalcluster.Name, groupResource schema.GroupResource) string { diff --git a/pkg/indexers/apiexport.go b/pkg/indexers/apiexport.go index 6b8e43241e2..36fc45a9a8d 100644 --- a/pkg/indexers/apiexport.go +++ b/pkg/indexers/apiexport.go @@ -19,6 +19,7 @@ package indexers import ( "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" kcpcache "github.com/kcp-dev/apimachinery/v2/pkg/cache" @@ -38,6 +39,9 @@ const ( APIExportByClaimedIdentities = "APIExportByClaimedIdentities" // APIExportEndpointSliceByAPIExport is the indexer name for retrieving APIExportEndpointSlices by their APIExport's Reference Path and Name. APIExportEndpointSliceByAPIExport = "APIExportEndpointSliceByAPIExport" + + APIExportByVirtualResourceIdentities = "APIExportByVirtualResourceIdentities" + APIExportByVirtualResourceIdentitiesAndGRs = "APIExportByVirtualResourceIdentitiesAndGRs" ) // IndexAPIExportByIdentity is an index function that indexes an APIExport by its identity hash. @@ -97,3 +101,48 @@ func IndexAPIExportEndpointSliceByAPIExport(obj interface{}) ([]string, error) { return result, nil } + +// IndexAPIExportByVirtualResourceIdentities is an index function that indexes an APIExport by its +// exported resources' virtual storage identity. +func IndexAPIExportByVirtualResourceIdentities(obj interface{}) ([]string, error) { + apiExport, ok := obj.(*apisv1alpha2.APIExport) + if !ok { + return []string{}, fmt.Errorf("obj %T is not an APIExport", obj) + } + + virtualResourceIdentities := sets.New[string]() + + for _, res := range apiExport.Spec.Resources { + if res.Storage.Virtual != nil { + virtualResourceIdentities.Insert(res.Storage.Virtual.IdentityHash) + } + } + + return sets.List[string](virtualResourceIdentities), nil +} + +// IndexAPIExportByVirtualResourceIdentities is an index function that indexes an APIExport by its +// exported resources' virtual storage identity and group resource. +func IndexAPIExportByVirtualResourceIdentitiesAndGRs(obj interface{}) ([]string, error) { + apiExport, ok := obj.(*apisv1alpha2.APIExport) + if !ok { + return []string{}, fmt.Errorf("obj %T is not an APIExport", obj) + } + + keys := sets.New[string]() + + for _, res := range apiExport.Spec.Resources { + if res.Storage.Virtual != nil { + keys.Insert(VirtualResourceIdentityAndGRKey(res.Storage.Virtual.IdentityHash, schema.GroupResource{ + Group: res.Group, + Resource: res.Name, + })) + } + } + + return sets.List[string](keys), nil +} + +func VirtualResourceIdentityAndGRKey(identityHash string, gr schema.GroupResource) string { + return fmt.Sprintf("%s:%s", gr.String(), identityHash) +} diff --git a/pkg/reconciler/apis/apibinding/apibinding_reconcile.go b/pkg/reconciler/apis/apibinding/apibinding_reconcile.go index 14577a2d544..0447e9f9cb5 100644 --- a/pkg/reconciler/apis/apibinding/apibinding_reconcile.go +++ b/pkg/reconciler/apis/apibinding/apibinding_reconcile.go @@ -502,14 +502,18 @@ func (r *bindingReconciler) reconcile(ctx context.Context, apiBinding *apisv1alp // Merge any current storage versions with new ones storageVersions := sets.New[string]() - if existingCRD != nil { - storageVersions.Insert(existingCRD.Status.StoredVersions...) - } + if resourceSchema.Storage.CRD != nil { + // Only resources with CRD storage need to track storage versions. - for _, b := range apiBinding.Status.BoundResources { - if b.Group == sch.Spec.Group && b.Resource == sch.Spec.Names.Plural { - storageVersions.Insert(b.StorageVersions...) - break + if existingCRD != nil { + storageVersions.Insert(existingCRD.Status.StoredVersions...) + } + + for _, b := range apiBinding.Status.BoundResources { + if b.Group == sch.Spec.Group && b.Resource == sch.Spec.Names.Plural { + storageVersions.Insert(b.StorageVersions...) + break + } } } diff --git a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_controller.go b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_controller.go index 38d0e716918..bc6077ba838 100644 --- a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_controller.go +++ b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_controller.go @@ -24,8 +24,10 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" @@ -37,16 +39,15 @@ import ( "github.com/kcp-dev/kcp/pkg/indexers" "github.com/kcp-dev/kcp/pkg/logging" "github.com/kcp-dev/kcp/pkg/reconciler/committer" + "github.com/kcp-dev/kcp/pkg/reconciler/events" "github.com/kcp-dev/kcp/pkg/tombstone" - apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" "github.com/kcp-dev/kcp/sdk/apis/core" - corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + topologyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/topology/v1alpha1" kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" cachev1alpha1client "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/typed/cache/v1alpha1" - apisv1alpha2informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha2" cachev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/cache/v1alpha1" - corev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/core/v1alpha1" + topologyinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/topology/v1alpha1" ) const ( @@ -55,12 +56,9 @@ const ( // NewController returns a new controller for CachedResourceEndpointSlices. func NewController( - shardName string, cachedResourceEndpointSliceClusterInformer cachev1alpha1informers.CachedResourceEndpointSliceClusterInformer, - cachedResourceClusterInformer cachev1alpha1informers.CachedResourceClusterInformer, - globalShardClusterInformer corev1alpha1informers.ShardClusterInformer, - lcClusterInformer corev1alpha1informers.LogicalClusterClusterInformer, - apiBindingClusterInformer apisv1alpha2informers.APIBindingClusterInformer, + globalCachedResourceClusterInformer cachev1alpha1informers.CachedResourceClusterInformer, + partitionClusterInformer topologyinformers.PartitionClusterInformer, kcpClusterClient kcpclientset.ClusterInterface, ) (*controller, error) { c := &controller{ @@ -70,32 +68,28 @@ func NewController( Name: ControllerName, }, ), - listCachedResourceEndpointSlicesByCachedResource: func(cachedResource *cachev1alpha1.CachedResource) ([]*cachev1alpha1.CachedResourceEndpointSlice, error) { - return indexers.ByIndex[*cachev1alpha1.CachedResourceEndpointSlice]( - cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), - byCachedResourceAndLogicalCluster, - cachedResourceEndpointSliceByCachedResourceAndLogicalCluster( - &cachev1alpha1.CachedResourceReference{ - Name: cachedResource.Name, - }, - logicalcluster.From(cachedResource), - ), - ) + listCachedResourceEndpointSlices: func() ([]*cachev1alpha1.CachedResourceEndpointSlice, error) { + return cachedResourceEndpointSliceClusterInformer.Lister().List(labels.Everything()) }, - getCachedResourceEndpointSlice: func(clusterName logicalcluster.Name, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) { - return cachedResourceEndpointSliceClusterInformer.Cluster(clusterName).Lister().Get(name) + getCachedResourceEndpointSlice: func(path logicalcluster.Path, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) { + return indexers.ByPathAndName[*cachev1alpha1.CachedResourceEndpointSlice](cachev1alpha1.Resource("cachedresourceendpointslice"), cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), path, name) }, - getCachedResource: func(clusterName logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { - return cachedResourceClusterInformer.Cluster(clusterName).Lister().Get(name) + getCachedResource: func(path logicalcluster.Path, name string) (*cachev1alpha1.CachedResource, error) { + return indexers.ByPathAndName[*cachev1alpha1.CachedResource](cachev1alpha1.Resource("cachedresource"), globalCachedResourceClusterInformer.Informer().GetIndexer(), path, name) }, - getMyShard: func() (*corev1alpha1.Shard, error) { - return globalShardClusterInformer.Cluster(core.RootCluster).Lister().Get(shardName) + getPartition: func(clusterName logicalcluster.Name, name string) (*topologyv1alpha1.Partition, error) { + return partitionClusterInformer.Lister().Cluster(clusterName).Get(name) }, - getLogicalCluster: func(clusterName logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) { - return lcClusterInformer.Cluster(clusterName).Lister().Get("cluster") - }, - getAPIBinding: func(clusterName logicalcluster.Name, bindingName string) (*apisv1alpha2.APIBinding, error) { - return apiBindingClusterInformer.Cluster(clusterName).Lister().Get(bindingName) + getCachedResourceEndpointSlicesByPartition: func(key string) ([]*cachev1alpha1.CachedResourceEndpointSlice, error) { + list, err := cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer().ByIndex(indexCachedResourceEndpointSlicesByPartition, key) + if err != nil { + return nil, err + } + slices := make([]*cachev1alpha1.CachedResourceEndpointSlice, 0, len(list)) + for _, obj := range list { + slices = append(slices, obj.(*cachev1alpha1.CachedResourceEndpointSlice)) + } + return slices, nil }, cachedResourceEndpointSliceClusterInformer: cachedResourceEndpointSliceClusterInformer, commit: committer.NewCommitter[*CachedResourceEndpointSlice, Patcher, *CachedResourceEndpointSliceSpec, *CachedResourceEndpointSliceStatus](kcpClusterClient.CacheV1alpha1().CachedResourceEndpointSlices()), @@ -105,27 +99,36 @@ func NewController( _, _ = cachedResourceEndpointSliceClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger) + c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger, "") }, UpdateFunc: func(_, newObj interface{}) { - c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](newObj), logger) + c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](newObj), logger, "") }, DeleteFunc: func(obj interface{}) { - c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger) + c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger, "") }, }) - _, _ = cachedResourceClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, _ = globalCachedResourceClusterInformer.Informer().AddEventHandler(events.WithoutSyncs(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { - c.enqueueCachedResourceEndpointSliceByCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), logger) + c.enqueueCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), logger) + }, + DeleteFunc: func(obj interface{}) { + c.enqueueCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), logger) }, - UpdateFunc: func(oldObj, newObj interface{}) { - c.enqueueCachedResourceEndpointSliceByCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](newObj), logger) + })) + + _, _ = partitionClusterInformer.Informer().AddEventHandler(events.WithoutSyncs(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueuePartition(tombstone.Obj[*topologyv1alpha1.Partition](obj), logger) + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueuePartition(tombstone.Obj[*topologyv1alpha1.Partition](newObj), logger) }, DeleteFunc: func(obj interface{}) { - c.enqueueCachedResourceEndpointSliceByCachedResource(tombstone.Obj[*cachev1alpha1.CachedResource](obj), logger) + c.enqueuePartition(tombstone.Obj[*topologyv1alpha1.Partition](obj), logger) }, - }) + })) return c, nil } @@ -142,39 +145,73 @@ type CommitFunc = func(context.Context, *Resource, *Resource) error type controller struct { queue workqueue.TypedRateLimitingInterface[string] - listCachedResourceEndpointSlicesByCachedResource func(cachedResource *cachev1alpha1.CachedResource) ([]*cachev1alpha1.CachedResourceEndpointSlice, error) - getCachedResourceEndpointSlice func(clusterName logicalcluster.Name, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) - getCachedResource func(clusterName logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) - getMyShard func() (*corev1alpha1.Shard, error) - getLogicalCluster func(clusterName logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) - getAPIBinding func(clusterName logicalcluster.Name, bindingName string) (*apisv1alpha2.APIBinding, error) + listCachedResourceEndpointSlices func() ([]*cachev1alpha1.CachedResourceEndpointSlice, error) + getCachedResourceEndpointSlice func(path logicalcluster.Path, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) + getCachedResource func(path logicalcluster.Path, name string) (*cachev1alpha1.CachedResource, error) + getPartition func(cluster logicalcluster.Name, name string) (*topologyv1alpha1.Partition, error) + getCachedResourceEndpointSlicesByPartition func(key string) ([]*cachev1alpha1.CachedResourceEndpointSlice, error) cachedResourceEndpointSliceClusterInformer cachev1alpha1informers.CachedResourceEndpointSliceClusterInformer commit CommitFunc } -// enqueueCachedResourceEndpointSlice enqueues an CachedResourceEndpointSlice. -func (c *controller) enqueueCachedResourceEndpointSlice(endpoints *cachev1alpha1.CachedResourceEndpointSlice, logger logr.Logger) { +func (c *controller) enqueueCachedResourceEndpointSlice(endpoints *cachev1alpha1.CachedResourceEndpointSlice, logger logr.Logger, logSuffix string) { key, err := kcpcache.DeletionHandlingMetaClusterNamespaceKeyFunc(endpoints) if err != nil { utilruntime.HandleError(err) return } - logger.V(4).Info("queueing CachedResourceEndpointSlice") + logger.V(4).Info(fmt.Sprintf("queueing CachedResourceEndpointSlice%s", logSuffix)) c.queue.Add(key) } -func (c *controller) enqueueCachedResourceEndpointSliceByCachedResource(cachedResource *cachev1alpha1.CachedResource, logger logr.Logger) { - slices, err := c.listCachedResourceEndpointSlicesByCachedResource(cachedResource) +func (c *controller) enqueueCachedResource(cachedResource *cachev1alpha1.CachedResource, logger logr.Logger) { + // binding keys by full path + keys := sets.New[string]() + if path := logicalcluster.NewPath(cachedResource.Annotations[core.LogicalClusterPathAnnotationKey]); !path.Empty() { + pathKeys, err := c.cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer().IndexKeys(IndexCachedResourceEndpointSliceByCachedResource, path.Join(cachedResource.Name).String()) + if err != nil { + utilruntime.HandleError(err) + return + } + keys.Insert(pathKeys...) + } + + clusterKeys, err := c.cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer().IndexKeys(IndexCachedResourceEndpointSliceByCachedResource, logicalcluster.From(cachedResource).Path().Join(cachedResource.Name).String()) + if err != nil { + utilruntime.HandleError(err) + return + } + keys.Insert(clusterKeys...) + + for _, key := range sets.List[string](keys) { + slice, exists, err := c.cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer().GetByKey(key) + if err != nil { + utilruntime.HandleError(err) + continue + } else if !exists { + continue + } + c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](slice), logger, " because of referenced CachedResource") + } +} + +func (c *controller) enqueuePartition(obj *topologyv1alpha1.Partition, logger logr.Logger) { + key, err := kcpcache.DeletionHandlingMetaClusterNamespaceKeyFunc(obj) if err != nil { utilruntime.HandleError(err) return } - for i := range slices { - logger.V(4).Info("queueing CachedResourceEndpointSlice because of CachedResource") - c.enqueueCachedResourceEndpointSlice(slices[i], logger) + slices, err := c.getCachedResourceEndpointSlicesByPartition(key) + if err != nil { + utilruntime.HandleError(err) + return + } + + for _, slice := range slices { + c.enqueueCachedResourceEndpointSlice(tombstone.Obj[*cachev1alpha1.CachedResourceEndpointSlice](slice), logger, " because of Partition change") } } @@ -234,7 +271,7 @@ func (c *controller) process(ctx context.Context, key string) (bool, error) { utilruntime.HandleError(err) return false, nil } - obj, err := c.getCachedResourceEndpointSlice(clusterName, name) + obj, err := c.getCachedResourceEndpointSlice(clusterName.Path(), name) if err != nil { if errors.IsNotFound(err) { return false, nil // object deleted before we handled it @@ -270,9 +307,15 @@ func (c *controller) process(ctx context.Context, key string) (bool, error) { // InstallIndexers adds the additional indexers that this controller requires to the informers. func InstallIndexers( + globalCachedResourceClusterInformer cachev1alpha1informers.CachedResourceClusterInformer, cachedResourceEndpointSliceClusterInformer cachev1alpha1informers.CachedResourceEndpointSliceClusterInformer, ) { + indexers.AddIfNotPresentOrDie(globalCachedResourceClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + }) indexers.AddIfNotPresentOrDie(cachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), cache.Indexers{ - byCachedResourceAndLogicalCluster: indexCachedResourceEndpointSliceByCachedResourceAndLogicalCluster, + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + IndexCachedResourceEndpointSliceByCachedResource: IndexCachedResourceEndpointSliceByCachedResourceFunc, + indexCachedResourceEndpointSlicesByPartition: indexCachedResourceEndpointSlicesByPartitionFunc, }) } diff --git a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_indexers.go b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_indexers.go index 9eb5b06e2d9..935650f7d3b 100644 --- a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_indexers.go +++ b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_indexers.go @@ -22,28 +22,45 @@ import ( "github.com/kcp-dev/logicalcluster/v3" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" + "github.com/kcp-dev/kcp/sdk/client" ) const ( - byCachedResourceAndLogicalCluster = "byCachedResourceAndLogicalCluster" + indexCachedResourceEndpointSlicesByPartition = "indexCachedResourceEndpointSlicesByPartition" + + IndexCachedResourceEndpointSliceByCachedResource = "IndexCachedResourceEndpointSliceByCachedResource" ) -// indexCachedResourceEndpointSliceByCachedResourceAndLogicalCluster is an index function that maps a reference to CachedResource and its cluster to a key. -func indexCachedResourceEndpointSliceByCachedResourceAndLogicalCluster(obj interface{}) ([]string, error) { - endpoints, ok := obj.(*cachev1alpha1.CachedResourceEndpointSlice) +// indexCachedResourceEndpointSlicesByPartitionFunc is an index function that maps a Partition to the key for its +// spec.partition. +func indexCachedResourceEndpointSlicesByPartitionFunc(obj interface{}) ([]string, error) { + slice, ok := obj.(*cachev1alpha1.CachedResourceEndpointSlice) if !ok { - return []string{}, fmt.Errorf("obj is supposed to be CachedResourceEndpointSlice, but is %T", obj) + return []string{}, fmt.Errorf("obj is supposed to be an CachedResourceEndpointSlice, but is %T", obj) + } + + if slice.Spec.Partition != "" { + clusterName := logicalcluster.From(slice).Path() + if !ok { + // this will never happen due to validation + return []string{}, fmt.Errorf("cluster information missing") + } + key := client.ToClusterAwareKey(clusterName, slice.Spec.Partition) + return []string{key}, nil } - key := cachedResourceEndpointSliceByCachedResourceAndLogicalCluster(&endpoints.Spec.CachedResource, logicalcluster.From(endpoints)) - return []string{key}, nil + return []string{}, nil } -func cachedResourceEndpointSliceByCachedResourceAndLogicalCluster(cachedResourceRef *cachev1alpha1.CachedResourceReference, cluster logicalcluster.Name) string { - var key string - key += cachedResourceRef.Name - if !cluster.Empty() { - key += "|" + cluster.String() +// IndexCachedResourceEndpointSliceByCachedResourceFunc is an index function that indexes +// a CachedResourceEndpointSlice by the CachedResource it references. +func IndexCachedResourceEndpointSliceByCachedResourceFunc(obj interface{}) ([]string, error) { + slice, ok := obj.(*cachev1alpha1.CachedResourceEndpointSlice) + if !ok { + return []string{}, fmt.Errorf("obj %T is not an CachedResourceEndpointSlice", obj) } - return key + + pathLocal := logicalcluster.From(slice).Path() + // TODO(gman0): add an optional external path index key once we add "CachedResourceEndpointSlice.spec.cachedResource.path". + return []string{pathLocal.Join(slice.Spec.CachedResource.Name).String()}, nil } diff --git a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile.go b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile.go index de8024c47bb..c224527ae4a 100644 --- a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile.go +++ b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile.go @@ -18,137 +18,115 @@ package cachedresourceendpointslice import ( "context" - "net/url" - "path" apierrors "k8s.io/apimachinery/pkg/api/errors" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/klog/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "github.com/kcp-dev/logicalcluster/v3" - virtualworkspacesoptions "github.com/kcp-dev/kcp/cmd/virtual-workspaces/options" - "github.com/kcp-dev/kcp/pkg/logging" - apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" + topologyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/topology/v1alpha1" ) -type reconcileStatus int - -const ( - reconcileStatusContinue reconcileStatus = iota - reconcileStatusStopAndRequeue - reconcileStatusStop -) - -type reconciler interface { - reconcile(ctx context.Context, endpoints *cachev1alpha1.CachedResourceEndpointSlice) (reconcileStatus, error) -} - func (c *controller) reconcile(ctx context.Context, endpoints *cachev1alpha1.CachedResourceEndpointSlice) (bool, error) { - reconcilers := []reconciler{ - &endpointsReconciler{ - getLogicalCluster: c.getLogicalCluster, - getAPIBinding: c.getAPIBinding, - getCachedResource: c.getCachedResource, - getMyShard: c.getMyShard, - }, - } - - var errs []error - - requeue := false - for _, r := range reconcilers { - var err error - var status reconcileStatus - status, err = r.reconcile(ctx, endpoints) - if err != nil { - errs = append(errs, err) - } - if status == reconcileStatusStopAndRequeue { - requeue = true - break - } - if status == reconcileStatusStop { - break - } + r := &endpointsReconciler{ + getCachedResource: c.getCachedResource, + getPartition: c.getPartition, } - return requeue, utilerrors.NewAggregate(errs) + return r.reconcile(ctx, endpoints) } type endpointsReconciler struct { - getLogicalCluster func(clusterName logicalcluster.Name) (*corev1alpha1.LogicalCluster, error) - getAPIBinding func(clusterName logicalcluster.Name, bindingName string) (*apisv1alpha2.APIBinding, error) - getCachedResource func(clusterName logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) - getMyShard func() (*corev1alpha1.Shard, error) + getCachedResource func(path logicalcluster.Path, name string) (*cachev1alpha1.CachedResource, error) + getPartition func(clusterName logicalcluster.Name, name string) (*topologyv1alpha1.Partition, error) } -func (r *endpointsReconciler) reconcile(ctx context.Context, endpoints *cachev1alpha1.CachedResourceEndpointSlice) (reconcileStatus, error) { - logger := klog.FromContext(ctx) - - shard, err := r.getMyShard() +func (r *endpointsReconciler) reconcile(ctx context.Context, endpoints *cachev1alpha1.CachedResourceEndpointSlice) (bool, error) { + _, err := r.getCachedResource(logicalcluster.From(endpoints).Path(), endpoints.Spec.CachedResource.Name) if err != nil { - return reconcileStatusStopAndRequeue, err - } - - addr, err := url.Parse(shard.Spec.VirtualWorkspaceURL) - if err != nil { - // Should never happen - logger = logging.WithObject(logger, shard) - logger.Error( - err, "error parsing shard.spec.virtualWorkspaceURL", - "VirtualWorkspaceURL", shard.Spec.VirtualWorkspaceURL, - ) - return reconcileStatusStop, nil + if apierrors.IsNotFound(err) { + // Don't keep the endpoints if the CachedResource has been deleted. + endpoints.Status.CachedResourceEndpoints = nil + conditions.MarkFalse( + endpoints, + cachev1alpha1.CachedResourceValid, + cachev1alpha1.CachedResourceNotFoundReason, + conditionsv1alpha1.ConditionSeverityError, + "Error getting CachedResource %s|%s", + logicalcluster.From(endpoints), + endpoints.Spec.CachedResource.Name, + ) + // No need to try again. + return false, nil + } else { + conditions.MarkFalse( + endpoints, + cachev1alpha1.CachedResourceValid, + cachev1alpha1.InternalErrorReason, + conditionsv1alpha1.ConditionSeverityError, + "Error getting CachedResource %s|%s", + logicalcluster.From(endpoints), + endpoints.Spec.CachedResource.Name, + ) + return true, err + } } + conditions.MarkTrue(endpoints, cachev1alpha1.CachedResourceValid) - // TODO(gmna0): this needs handling for per-shard URLs. To be completed - // once we do CachedResource aggregation with APIExports. - - // Formats the Replication VW URL like so: - // /services/replication// - addr.Path = path.Join( - addr.Path, - virtualworkspacesoptions.DefaultRootPathPrefix, - "replication", - logicalcluster.From(endpoints).String(), - endpoints.Spec.CachedResource.Name, - ) - - addrUrl := addr.String() - - _, err = r.getCachedResource(logicalcluster.From(endpoints), endpoints.Spec.CachedResource.Name) - if err == nil { - endpoints.Status.CachedResourceEndpoints = addURLIfNotPresent(endpoints.Status.CachedResourceEndpoints, addrUrl) - return reconcileStatusContinue, nil + // Check the partition selector. + var selector labels.Selector + if endpoints.Spec.Partition != "" { + partition, err := r.getPartition(logicalcluster.From(endpoints), endpoints.Spec.Partition) + if err != nil { + if apierrors.IsNotFound(err) { + // Don't keep the endpoints if the Partition has been deleted and is still referenced. + endpoints.Status.CachedResourceEndpoints = nil + conditions.MarkFalse( + endpoints, + cachev1alpha1.PartitionValid, + cachev1alpha1.PartitionInvalidReferenceReason, + conditionsv1alpha1.ConditionSeverityError, + "%v", + err, + ) + // No need to try again. + return false, nil + } else { + conditions.MarkFalse( + endpoints, + cachev1alpha1.PartitionValid, + cachev1alpha1.InternalErrorReason, + conditionsv1alpha1.ConditionSeverityError, + "%v", + err, + ) + return true, err + } + } + selector, err = metav1.LabelSelectorAsSelector(partition.Spec.Selector) + if err != nil { + conditions.MarkFalse( + endpoints, + cachev1alpha1.PartitionValid, + cachev1alpha1.PartitionInvalidReferenceReason, + conditionsv1alpha1.ConditionSeverityError, + "%v", + err, + ) + return true, err + } } - if apierrors.IsNotFound(err) { - endpoints.Status.CachedResourceEndpoints = removeURLIfPresent(endpoints.Status.CachedResourceEndpoints, addrUrl) - return reconcileStatusContinue, nil + if selector == nil { + selector = labels.Everything() } - return reconcileStatusStopAndRequeue, err -} + conditions.MarkTrue(endpoints, cachev1alpha1.PartitionValid) -func addURLIfNotPresent(endpoints []cachev1alpha1.CachedResourceEndpoint, urlToAdd string) []cachev1alpha1.CachedResourceEndpoint { - for _, endpoint := range endpoints { - if endpoint.URL == urlToAdd { - // Already in endpoints slice, nothing to do. - return endpoints - } - } - return append(endpoints, cachev1alpha1.CachedResourceEndpoint{ - URL: urlToAdd, - }) -} + endpoints.Status.ShardSelector = selector.String() -func removeURLIfPresent(endpoints []cachev1alpha1.CachedResourceEndpoint, urlToRemove string) []cachev1alpha1.CachedResourceEndpoint { - for i, endpoint := range endpoints { - if endpoint.URL == urlToRemove { - return append(endpoints[:i], endpoints[i+1:]...) - } - } - return endpoints + return true, err } diff --git a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile_test.go b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile_test.go index 5b18a3ced79..058cd630611 100644 --- a/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile_test.go +++ b/pkg/reconciler/cache/cachedresourceendpointslice/cachedresourceendpointslice_reconcile_test.go @@ -18,8 +18,10 @@ package cachedresourceendpointslice import ( "context" + "fmt" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -28,233 +30,152 @@ import ( "github.com/kcp-dev/logicalcluster/v3" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" - corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" + topologyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/topology/v1alpha1" ) -func TestEndpointsReconciler(t *testing.T) { +func TestReconcile(t *testing.T) { tests := map[string]struct { - r endpointsReconciler - in, out cachev1alpha1.CachedResourceEndpointSlice - wantsErr error - wantsStatus reconcileStatus + keyMissing bool + cachedResourceMissing bool + partitionMissing bool + cachedResourceInternalErr bool + listShardsError error + errorReason string + + wantError bool + wantVerifyFailure bool + wantCachedResourceValid bool + wantPartitionValid bool + wantCachedResourceNotValid bool + wantPartitionNotValid bool }{ - "ShouldAdd": { - r: endpointsReconciler{ - getCachedResource: func(logicalcluster.Name, string) (*cachev1alpha1.CachedResource, error) { - return &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-cachedresource", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - }, nil - }, - getMyShard: func() (*corev1alpha1.Shard, error) { - return &corev1alpha1.Shard{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-shard", - }, - Spec: corev1alpha1.ShardSpec{ - VirtualWorkspaceURL: "https://my-shard.kcp.dev", - }, - }, nil - }, - }, - in: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, - }, - }, - out: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, - }, - Status: cachev1alpha1.CachedResourceEndpointSliceStatus{ - CachedResourceEndpoints: []cachev1alpha1.CachedResourceEndpoint{ - { - URL: "https://my-shard.kcp.dev/services/replication/my-workspace/my-cachedresource"}, - }, - }, - }, - wantsStatus: reconcileStatusContinue, + "CachedResourceValid set to false when CachedResource is missing": { + cachedResourceMissing: true, + errorReason: cachev1alpha1.CachedResourceNotFoundReason, + wantCachedResourceNotValid: true, }, - "ShouldNotAddBecauseAlreadyPresent": { - r: endpointsReconciler{ - getCachedResource: func(logicalcluster.Name, string) (*cachev1alpha1.CachedResource, error) { - return &cachev1alpha1.CachedResource{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-cachedresource", - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - }, nil - }, - getMyShard: func() (*corev1alpha1.Shard, error) { - return &corev1alpha1.Shard{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-shard", - }, - Spec: corev1alpha1.ShardSpec{ - VirtualWorkspaceURL: "https://my-shard.kcp.dev", - }, - }, nil - }, - }, - in: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, - }, - Status: cachev1alpha1.CachedResourceEndpointSliceStatus{ - CachedResourceEndpoints: []cachev1alpha1.CachedResourceEndpoint{ - { - URL: "https://my-shard.kcp.dev/services/replication/my-workspace/my-cachedresource"}, - }, - }, - }, - out: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, - }, - Status: cachev1alpha1.CachedResourceEndpointSliceStatus{ - CachedResourceEndpoints: []cachev1alpha1.CachedResourceEndpoint{ - { - URL: "https://my-shard.kcp.dev/services/replication/my-workspace/my-cachedresource"}, - }, - }, - }, - wantsStatus: reconcileStatusContinue, + "CachedResourceValid set to false if an internal error happens when fetching the CachedResource": { + cachedResourceInternalErr: true, + wantError: true, + errorReason: cachev1alpha1.InternalErrorReason, + wantCachedResourceNotValid: true, }, - "ShouldRemove": { - r: endpointsReconciler{ - getCachedResource: func(logicalcluster.Name, string) (*cachev1alpha1.CachedResource, error) { - return nil, apierrors.NewNotFound(cachev1alpha1.Resource("cachedresources"), "my-cachedresource") - }, - getMyShard: func() (*corev1alpha1.Shard, error) { - return &corev1alpha1.Shard{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-shard", - }, - Spec: corev1alpha1.ShardSpec{ - VirtualWorkspaceURL: "https://my-shard.kcp.dev", - }, - }, nil - }, - }, - in: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, - }, - Status: cachev1alpha1.CachedResourceEndpointSliceStatus{ - CachedResourceEndpoints: []cachev1alpha1.CachedResourceEndpoint{ - { - URL: "https://my-shard.kcp.dev/services/replication/my-workspace/my-cachedresource"}, - }, - }, - }, - out: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, - }, - Status: cachev1alpha1.CachedResourceEndpointSliceStatus{ - CachedResourceEndpoints: []cachev1alpha1.CachedResourceEndpoint{}, - }, - }, - wantsStatus: reconcileStatusContinue, + "PartitionValid set to false when the Partition is missing": { + partitionMissing: true, + errorReason: cachev1alpha1.PartitionInvalidReferenceReason, + wantPartitionNotValid: true, }, - "ShouldSkipBecauseNotFound": { - r: endpointsReconciler{ - getCachedResource: func(logicalcluster.Name, string) (*cachev1alpha1.CachedResource, error) { - return nil, apierrors.NewNotFound(cachev1alpha1.Resource("cachedresources"), "my-cachedresource") - }, - getMyShard: func() (*corev1alpha1.Shard, error) { - return &corev1alpha1.Shard{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-shard", - }, - Spec: corev1alpha1.ShardSpec{ - VirtualWorkspaceURL: "https://my-shard.kcp.dev", - }, - }, nil - }, - }, - in: cachev1alpha1.CachedResourceEndpointSlice{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", - }, - }, - Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ - CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", - }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + c := &controller{ + getCachedResource: func(path logicalcluster.Path, name string) (*cachev1alpha1.CachedResource, error) { + if tc.cachedResourceMissing { + return nil, apierrors.NewNotFound(cachev1alpha1.Resource("CachedResource"), name) + } else if tc.cachedResourceInternalErr { + return nil, fmt.Errorf("internal error") + } else { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + logicalcluster.AnnotationKey: "root:org:ws", + }, + Name: "my-cr", + }, + }, nil + } + }, + getPartition: func(clusterName logicalcluster.Name, name string) (*topologyv1alpha1.Partition, error) { + if tc.partitionMissing { + return nil, apierrors.NewNotFound(topologyv1alpha1.Resource("Partition"), name) + } else { + return &topologyv1alpha1.Partition{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + logicalcluster.AnnotationKey: "root:org:ws", + }, + Name: "my-partition", + }, + Spec: topologyv1alpha1.PartitionSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "region": "Europe", + }, + }, + }, + }, nil + } }, - }, - out: cachev1alpha1.CachedResourceEndpointSlice{ + } + + cachedResourceEndpointSlice := &cachev1alpha1.CachedResourceEndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - logicalcluster.AnnotationKey: "my-workspace", + logicalcluster.AnnotationKey: "root:org:ws", }, + Name: "my-slice", }, Spec: cachev1alpha1.CachedResourceEndpointSliceSpec{ CachedResource: cachev1alpha1.CachedResourceReference{ - Name: "my-cachedresource", + Name: "my-cr", }, + Partition: "my-partition", }, - }, - wantsStatus: reconcileStatusContinue, - }, - } + } + _, err := c.reconcile(context.Background(), cachedResourceEndpointSlice) + if tc.wantError { + require.Error(t, err, "expected an error") + } else { + require.NoError(t, err, "expected no error") + } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - status, err := tc.r.reconcile(context.Background(), &tc.in) - require.Equal(t, tc.wantsStatus, status, "unexpected reconcile status") - require.Equal(t, tc.wantsErr, err, "unexpected error value") - require.Equal(t, tc.out, tc.in) + if tc.wantCachedResourceNotValid { + requireConditionMatches(t, cachedResourceEndpointSlice, + conditions.FalseCondition( + cachev1alpha1.CachedResourceValid, + tc.errorReason, + conditionsv1alpha1.ConditionSeverityError, + "", + ), + ) + } + + if tc.wantPartitionNotValid { + requireConditionMatches(t, cachedResourceEndpointSlice, + conditions.FalseCondition( + cachev1alpha1.PartitionValid, + tc.errorReason, + conditionsv1alpha1.ConditionSeverityError, + "", + ), + ) + } + + if tc.wantCachedResourceValid { + requireConditionMatches(t, cachedResourceEndpointSlice, + conditions.TrueCondition(cachev1alpha1.CachedResourceValid), + ) + } + + if tc.wantPartitionValid { + requireConditionMatches(t, cachedResourceEndpointSlice, + conditions.TrueCondition(cachev1alpha1.PartitionValid), + ) + } }) } } + +// requireConditionMatches looks for a condition matching c in g. LastTransitionTime and Message +// are not compared. +func requireConditionMatches(t *testing.T, g conditions.Getter, c *conditionsv1alpha1.Condition) { + t.Helper() + actual := conditions.Get(g, c.Type) + require.NotNil(t, actual, "missing condition %q", c.Type) + actual.LastTransitionTime = c.LastTransitionTime + actual.Message = c.Message + require.Empty(t, cmp.Diff(actual, c)) +} diff --git a/pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_controller.go b/pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_controller.go new file mode 100644 index 00000000000..16724bb848e --- /dev/null +++ b/pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_controller.go @@ -0,0 +1,369 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachedresourceendpointsliceurls + +import ( + "context" + "fmt" + "time" + + "github.com/go-logr/logr" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "k8s.io/utils/ptr" + + kcpcache "github.com/kcp-dev/apimachinery/v2/pkg/cache" + "github.com/kcp-dev/logicalcluster/v3" + + "github.com/kcp-dev/kcp/pkg/indexers" + "github.com/kcp-dev/kcp/pkg/logging" + "github.com/kcp-dev/kcp/pkg/reconciler/events" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" + cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/core" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + cachev1alpha1apply "github.com/kcp-dev/kcp/sdk/client/applyconfiguration/cache/v1alpha1" + kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" + apisv1alpha2informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha2" + cachev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/cache/v1alpha1" + corev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/core/v1alpha1" +) + +const ( + ControllerName = "kcp-cachedresourceendpointslice-urls" +) + +func listAPIBindingsByAPIExport(apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer, export *apisv1alpha2.APIExport) ([]*apisv1alpha2.APIBinding, error) { + // binding keys by full path + keys := sets.New[string]() + if path := logicalcluster.NewPath(export.Annotations[core.LogicalClusterPathAnnotationKey]); !path.Empty() { + pathKeys, err := apiBindingInformer.Informer().GetIndexer().IndexKeys(indexers.APIBindingsByAPIExport, path.Join(export.Name).String()) + if err != nil { + return nil, err + } + keys.Insert(pathKeys...) + } + + clusterKeys, err := apiBindingInformer.Informer().GetIndexer().IndexKeys(indexers.APIBindingsByAPIExport, logicalcluster.From(export).Path().Join(export.Name).String()) + if err != nil { + return nil, err + } + keys.Insert(clusterKeys...) + + bindings := make([]*apisv1alpha2.APIBinding, 0, keys.Len()) + for _, key := range sets.List[string](keys) { + binding, exists, err := apiBindingInformer.Informer().GetIndexer().GetByKey(key) + if err != nil { + utilruntime.HandleError(err) + continue + } else if !exists { + utilruntime.HandleError(fmt.Errorf("APIBinding %q does not exist", key)) + continue + } + bindings = append(bindings, binding.(*apisv1alpha2.APIBinding)) + } + return bindings, nil +} + +func NewController( + shardName string, + apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer, + localCachedResourceEndpointSliceClusterInformer, globalCachedResourceEndpointSliceClusterInformer cachev1alpha1informers.CachedResourceEndpointSliceClusterInformer, + globalShardClusterInformer corev1alpha1informers.ShardClusterInformer, + localAPIExportClusterInformer, globalAPIExportClusterInformer apisv1alpha2informers.APIExportClusterInformer, + globalCachedResourcelusterInformer cachev1alpha1informers.CachedResourceClusterInformer, + globalLogicalClusterInformer corev1alpha1informers.LogicalClusterClusterInformer, + clusterClient kcpclientset.ClusterInterface, +) (*controller, error) { + c := &controller{ + shardName: shardName, + queue: workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{ + Name: ControllerName, + }, + ), + getMyShard: func() (*corev1alpha1.Shard, error) { + return globalShardClusterInformer.Cluster(core.RootCluster).Lister().Get(shardName) + }, + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return indexers.ByPathAndName[*cachev1alpha1.CachedResource](cachev1alpha1.Resource("cachedresources"), globalCachedResourcelusterInformer.Informer().GetIndexer(), cluster.Path(), name) + }, + getCachedResourceEndpointSlice: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) { + obj, err := indexers.ByPathAndNameWithFallback[*cachev1alpha1.CachedResourceEndpointSlice](cachev1alpha1.Resource("cachedresourceendpointslices"), localCachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), globalCachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), cluster.Path(), name) + if err != nil { + return nil, err + } + return obj, err + }, + listAPIExportsByCachedResourceIdentityAndGR: func(identityHash string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + apiExports, err := indexers.ByIndexWithFallback[*apisv1alpha2.APIExport](localAPIExportClusterInformer.Informer().GetIndexer(), globalAPIExportClusterInformer.Informer().GetIndexer(), indexers.APIExportByVirtualResourceIdentitiesAndGRs, indexers.VirtualResourceIdentityAndGRKey(identityHash, gr)) + if err != nil { + return nil, err + } + return apiExports, nil + }, + listAPIBindingsByAPIExports: func(exports []*apisv1alpha2.APIExport) ([]*apisv1alpha2.APIBinding, error) { + var bindings []*apisv1alpha2.APIBinding + for _, export := range exports { + bindingsForExport, err := listAPIBindingsByAPIExport(apiBindingInformer, export) + if err != nil { + return nil, err + } + bindings = append(bindings, bindingsForExport...) + } + return bindings, nil + }, + patchCachedResourceEndpointSlice: func(ctx context.Context, cluster logicalcluster.Path, patch *cachev1alpha1apply.CachedResourceEndpointSliceApplyConfiguration) error { + _, err := clusterClient.CacheV1alpha1().CachedResourceEndpointSlices().Cluster(cluster).ApplyStatus(ctx, patch, metav1.ApplyOptions{ + FieldManager: shardName, + }) + return err + }, + getAPIExport: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + return indexers.ByPathAndName[*apisv1alpha2.APIExport](apisv1alpha2.Resource("apiexports"), globalAPIExportClusterInformer.Informer().GetIndexer(), path, name) + }, + patchAPIExportEndpointSlice: func(ctx context.Context, cluster logicalcluster.Path, patch *cachev1alpha1apply.CachedResourceEndpointSliceApplyConfiguration) error { + _, err := clusterClient.CacheV1alpha1().CachedResourceEndpointSlices().Cluster(cluster).ApplyStatus(ctx, patch, metav1.ApplyOptions{ + FieldManager: shardName, + }) + return err + }, + } + + logger := logging.WithReconciler(klog.Background(), ControllerName) + + _, _ = apiBindingInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueAPIBinding(objOrTombstone[*apisv1alpha2.APIBinding](obj), logger) + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueueAPIBinding(objOrTombstone[*apisv1alpha2.APIBinding](newObj), logger) + }, + DeleteFunc: func(obj interface{}) { + c.enqueueAPIBinding(objOrTombstone[*apisv1alpha2.APIBinding](obj), logger) + }, + }) + + _, _ = localCachedResourceEndpointSliceClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueCachedResourceEndpointSlice(objOrTombstone[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger, "") + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueueCachedResourceEndpointSlice(objOrTombstone[*cachev1alpha1.CachedResourceEndpointSlice](newObj), logger, "") + }, + DeleteFunc: func(obj interface{}) { + c.enqueueCachedResourceEndpointSlice(objOrTombstone[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger, "") + }, + }) + + _, _ = globalCachedResourceEndpointSliceClusterInformer.Informer().AddEventHandler(events.WithoutSyncs(cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + c.enqueueCachedResourceEndpointSlice(objOrTombstone[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger, " from cache") + }, + UpdateFunc: func(_, newObj interface{}) { + c.enqueueCachedResourceEndpointSlice(objOrTombstone[*cachev1alpha1.CachedResourceEndpointSlice](newObj), logger, " from cache") + }, + DeleteFunc: func(obj interface{}) { + c.enqueueCachedResourceEndpointSlice(objOrTombstone[*cachev1alpha1.CachedResourceEndpointSlice](obj), logger, " from cache") + }, + })) + + return c, nil +} + +func (c *controller) enqueueAPIBinding(obj *apisv1alpha2.APIBinding, logger logr.Logger) { + exportPath := logicalcluster.NewPath(obj.Spec.Reference.Export.Path) + if exportPath.Empty() { + exportPath = logicalcluster.From(obj).Path() + } + export, err := c.getAPIExport(exportPath, obj.Spec.Reference.Export.Name) + if err != nil { + utilruntime.HandleError(err) + return + } + + logger = logging.WithObject(logger, obj) + + for _, resource := range export.Spec.Resources { + if resource.Storage.Virtual == nil || + ptr.Deref(resource.Storage.Virtual.Reference.APIGroup, "") != cachev1alpha1.SchemeGroupVersion.Group || + resource.Storage.Virtual.Reference.Kind != "CachedResourceEndpointSlice" { + logger.V(4).Info("skipping APIBinding its referenced APIExport does not export CachedResourceEndpointSlice virtual resources") + continue + } + + key := kcpcache.ToClusterAwareKey(logicalcluster.From(export).String(), "", resource.Storage.Virtual.Reference.Name) + logger.V(4).Info("queueing CachedResourceEndpointSlice because of APIBinding", "key", key) + c.queue.Add(key) + } +} + +func (c *controller) enqueueCachedResourceEndpointSlice(obj *cachev1alpha1.CachedResourceEndpointSlice, logger logr.Logger, logSuffix string) { + logger = logging.WithObject(logger, obj) + key, err := kcpcache.DeletionHandlingMetaClusterNamespaceKeyFunc(obj) + if err != nil { + utilruntime.HandleError(err) + return + } + + logger.V(4).Info("queueing CachedResourceEndpointSlice", "key", key) + c.queue.Add(key) +} + +type controller struct { + queue workqueue.TypedRateLimitingInterface[string] + shardName string + + listAPIExportsByCachedResourceIdentityAndGR func(identityHash string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) + listAPIBindingsByAPIExports func(exports []*apisv1alpha2.APIExport) ([]*apisv1alpha2.APIBinding, error) + getMyShard func() (*corev1alpha1.Shard, error) + getCachedResource func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) + getCachedResourceEndpointSlice func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResourceEndpointSlice, error) + patchAPIExportEndpointSlice func(ctx context.Context, cluster logicalcluster.Path, patch *cachev1alpha1apply.CachedResourceEndpointSliceApplyConfiguration) error + patchCachedResourceEndpointSlice func(ctx context.Context, cluster logicalcluster.Path, patch *cachev1alpha1apply.CachedResourceEndpointSliceApplyConfiguration) error + getAPIExport func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) +} + +// Start starts the controller, which stops when ctx.Done() is closed. +func (c *controller) Start(ctx context.Context, numThreads int) { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + logger := logging.WithReconciler(klog.FromContext(ctx), ControllerName) + ctx = klog.NewContext(ctx, logger) + logger.Info("Starting controller") + defer logger.Info("Shutting down controller") + + for range numThreads { + go wait.UntilWithContext(ctx, c.startWorker, time.Second) + } + + <-ctx.Done() +} + +func (c *controller) startWorker(ctx context.Context) { + for c.processNextWorkItem(ctx) { + } +} + +func (c *controller) processNextWorkItem(ctx context.Context) bool { + // Wait until there is a new item in the working queue + k, quit := c.queue.Get() + if quit { + return false + } + key := k + + logger := logging.WithQueueKey(klog.FromContext(ctx), key) + ctx = klog.NewContext(ctx, logger) + logger.V(4).Info("processing key") + + // No matter what, tell the queue we're done with this key, to unblock + // other workers. + defer c.queue.Done(key) + + if requeue, err := c.process(ctx, key); err != nil { + utilruntime.HandleError(fmt.Errorf("%q controller failed to sync %q, err: %w", ControllerName, key, err)) + c.queue.AddRateLimited(key) + return true + } else if requeue { + // only requeue if we didn't error, but we still want to requeue + c.queue.Add(key) + return true + } + c.queue.Forget(key) + return true +} + +func (c *controller) process(ctx context.Context, key string) (bool, error) { + cluster, _, name, err := kcpcache.SplitMetaClusterNamespaceKey(key) + if err != nil { + utilruntime.HandleError(err) + return false, nil + } + obj, err := c.getCachedResourceEndpointSlice(cluster, name) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil // object deleted before we handled it + } + return false, err + } + + obj = obj.DeepCopy() + + logger := logging.WithObject(klog.FromContext(ctx), obj) + ctx = klog.NewContext(ctx, logger) + + var errs []error + requeue, err := c.reconcile(ctx, obj) + if err != nil { + errs = append(errs, err) + } + + return requeue, utilerrors.NewAggregate(errs) +} + +func InstallIndexers( + localCachedResourceClusterInformer, globalCachedResourceClusterInformer cachev1alpha1informers.CachedResourceClusterInformer, + localCachedResourceEndpointSliceClusterInformer, globalCachedResourceEndpointSliceClusterInformer cachev1alpha1informers.CachedResourceEndpointSliceClusterInformer, + localAPIExportClusterInformer, globalAPIExportClusterInformer apisv1alpha2informers.APIExportClusterInformer, +) { + indexers.AddIfNotPresentOrDie(localAPIExportClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.APIExportByVirtualResourceIdentitiesAndGRs: indexers.IndexAPIExportByVirtualResourceIdentitiesAndGRs, + }) + indexers.AddIfNotPresentOrDie(globalAPIExportClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.APIExportByVirtualResourceIdentitiesAndGRs: indexers.IndexAPIExportByVirtualResourceIdentitiesAndGRs, + }) + indexers.AddIfNotPresentOrDie(localCachedResourceClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + }) + indexers.AddIfNotPresentOrDie(globalCachedResourceClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + }) + indexers.AddIfNotPresentOrDie(localCachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + }) + indexers.AddIfNotPresentOrDie(globalCachedResourceEndpointSliceClusterInformer.Informer().GetIndexer(), cache.Indexers{ + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + }) +} + +func objOrTombstone[T runtime.Object](obj any) T { + if t, ok := obj.(T); ok { + return t + } + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + if t, ok := tombstone.Obj.(T); ok { + return t + } + + panic(fmt.Errorf("tombstone %T is not a %T", tombstone, new(T))) + } + + panic(fmt.Errorf("%T is not a %T", obj, new(T))) +} diff --git a/pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_reconciler.go b/pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_reconciler.go new file mode 100644 index 00000000000..ebe532b06b9 --- /dev/null +++ b/pkg/reconciler/cache/cachedresourceendpointsliceurls/cachedresourceendpointsliceurls_reconciler.go @@ -0,0 +1,176 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachedresourceendpointsliceurls + +import ( + "context" + "net/url" + "path" + + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" + + "github.com/kcp-dev/logicalcluster/v3" + + virtualworkspacesoptions "github.com/kcp-dev/kcp/cmd/virtual-workspaces/options" + "github.com/kcp-dev/kcp/pkg/logging" + replicationvw "github.com/kcp-dev/kcp/pkg/virtual/replication" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" + cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" + corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" + cachev1alpha1apply "github.com/kcp-dev/kcp/sdk/client/applyconfiguration/cache/v1alpha1" +) + +type result struct { + url string + remove bool +} + +func (c *controller) reconcile(ctx context.Context, slice *cachev1alpha1.CachedResourceEndpointSlice) (bool, error) { + return endpointsReconciler{ + listAPIExportsByCachedResourceIdentityAndGR: c.listAPIExportsByCachedResourceIdentityAndGR, + listAPIBindingsByAPIExports: c.listAPIBindingsByAPIExports, + getMyShard: c.getMyShard, + getCachedResource: c.getCachedResource, + patchAPIExportEndpointSlice: c.patchAPIExportEndpointSlice, + }.reconcile(ctx, slice) +} + +type endpointsReconciler struct { + listAPIExportsByCachedResourceIdentityAndGR func(identityHash string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) + listAPIBindingsByAPIExports func(exports []*apisv1alpha2.APIExport) ([]*apisv1alpha2.APIBinding, error) + getMyShard func() (*corev1alpha1.Shard, error) + getCachedResource func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) + patchAPIExportEndpointSlice func(ctx context.Context, cluster logicalcluster.Path, patch *cachev1alpha1apply.CachedResourceEndpointSliceApplyConfiguration) error +} + +func (r endpointsReconciler) reconcile(ctx context.Context, slice *cachev1alpha1.CachedResourceEndpointSlice) (bool, error) { + for _, condition := range slice.Status.Conditions { + if !conditions.IsTrue(slice, condition.Type) { + return false, nil + } + } + + rs, err := r.updateEndpoints(ctx, slice) + if err != nil { + return true, err + } + if rs == nil { + // No change, nothing to do. + return false, nil + } + + // Patch the object + patch := cachev1alpha1apply.CachedResourceEndpointSlice(slice.Name) + if rs.remove { + patch.WithStatus(cachev1alpha1apply.CachedResourceEndpointSliceStatus()) + } else { + patch.WithStatus(cachev1alpha1apply.CachedResourceEndpointSliceStatus(). + WithCachedResourceEndpoints(cachev1alpha1apply.CachedResourceEndpoint().WithURL(rs.url))) + } + cluster := logicalcluster.From(slice) + err = r.patchAPIExportEndpointSlice(ctx, cluster.Path(), patch) + if err != nil { + return true, err + } + return false, nil +} + +func (r *endpointsReconciler) updateEndpoints(ctx context.Context, slice *cachev1alpha1.CachedResourceEndpointSlice) (*result, error) { + logger := klog.FromContext(ctx) + + thisShard, err := r.getMyShard() + if err != nil { + return nil, err + } + + if thisShard.Spec.VirtualWorkspaceURL == "" { + // We don't have VW URLs, bail out. + return nil, nil + } + + cr, err := r.getCachedResource(logicalcluster.From(slice), slice.Spec.CachedResource.Name) + if err != nil { + return nil, err + } + + exports, err := r.listAPIExportsByCachedResourceIdentityAndGR(cr.Status.IdentityHash, schema.GroupResource{ + Group: cr.Spec.Group, + Resource: cr.Spec.Resource, + }) + if err != nil { + return nil, err + } + + bindings, err := r.listAPIBindingsByAPIExports(exports) + if err != nil { + return nil, err + } + + if len(bindings) == 0 { + // We don't have any consumers, so clean up all endpoints. + return &result{ + remove: true, + }, nil + } + + shardSelector, err := labels.Parse(slice.Status.ShardSelector) + if err != nil { + return nil, err + } + if !shardSelector.Matches(labels.Set(thisShard.Labels)) { + // We don't belong in the partition, so do nothing. + return nil, nil + } + + // Construct the Replication VW URL and try to add it to the slice. + + vwURL, err := url.Parse(thisShard.Spec.VirtualWorkspaceURL) + if err != nil { + logger = logging.WithObject(logger, thisShard) + logger.Error( + err, "error parsing shard.spec.virtualWorkspaceURL", + "VirtualWorkspaceURL", thisShard.Spec.VirtualWorkspaceURL, + ) + // Can't do much more... + return nil, nil + } + + // Formats the Replication VW URL like so: + // /services/replication// + vwURL.Path = path.Join( + vwURL.Path, + virtualworkspacesoptions.DefaultRootPathPrefix, + replicationvw.VirtualWorkspaceName, + logicalcluster.From(cr).String(), + cr.Name, + ) + completeVWAddr := vwURL.String() + + for _, u := range slice.Status.CachedResourceEndpoints { + if u.URL == completeVWAddr { + // VW URL already in the endpoint slice, nothing to do. + return nil, nil + } + } + + return &result{ + url: completeVWAddr, + }, nil +} diff --git a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_identity_test.go b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_identity_test.go index 2b57e606def..62fdef4d98b 100644 --- a/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_identity_test.go +++ b/pkg/reconciler/cache/cachedresources/cachedresources_reconcile_identity_test.go @@ -292,3 +292,10 @@ func TestReconcileIdentity(t *testing.T) { }) } } + +func resetLastTransitionTime(conditions conditionsv1alpha1.Conditions) { + // We don't care about LastTransitionTime. + for i := range conditions { + conditions[i].LastTransitionTime = metav1.Time{} + } +} diff --git a/pkg/reconciler/cache/cachedresources/replication/indexers.go b/pkg/reconciler/cache/cachedresources/replication/indexers.go index d11aa914406..39ee1b044ba 100644 --- a/pkg/reconciler/cache/cachedresources/replication/indexers.go +++ b/pkg/reconciler/cache/cachedresources/replication/indexers.go @@ -104,8 +104,11 @@ func IndexByGVRAndLogicalClusterAndNamespace(obj interface{}) ([]string, error) } namespace := labels[LabelKeyObjectOriginalNamespace] - key := GVRAndLogicalClusterAndNamespace(gvr, logicalcluster.From(a), namespace) - return []string{key}, nil + return []string{ + GVRAndLogicalClusterAndNamespace(gvr, logicalcluster.From(a), namespace), + // The namespace-squashing key is for the cases where the client wants to list/watch across all namespaces. + GVRAndLogicalClusterAndNamespace(gvr, logicalcluster.From(a), ""), + }, nil } // GVRAndLogicalClusterAndNamespace creates an index key from the given parameters. diff --git a/pkg/reconciler/cache/replication/replication_controller.go b/pkg/reconciler/cache/replication/replication_controller.go index a99e6696ab4..ec097338ff1 100644 --- a/pkg/reconciler/cache/replication/replication_controller.go +++ b/pkg/reconciler/cache/replication/replication_controller.go @@ -42,6 +42,7 @@ import ( "github.com/kcp-dev/kcp/pkg/logging" apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" + cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" "github.com/kcp-dev/kcp/sdk/apis/core" corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" tenancyv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" @@ -241,6 +242,16 @@ func InstallIndexers( Local: localKubeInformers.Admissionregistration().V1().ValidatingAdmissionPolicyBindings().Informer(), Global: globalKubeInformers.Admissionregistration().V1().ValidatingAdmissionPolicyBindings().Informer(), }, + cachev1alpha1.SchemeGroupVersion.WithResource("cachedresources"): { + Kind: "CachedResource", + Local: localKcpInformers.Cache().V1alpha1().CachedResources().Informer(), + Global: globalKcpInformers.Cache().V1alpha1().CachedResources().Informer(), + }, + cachev1alpha1.SchemeGroupVersion.WithResource("cachedresourceendpointslices"): { + Kind: "CachedResourceEndpointSlice", + Local: localKcpInformers.Cache().V1alpha1().CachedResourceEndpointSlices().Informer(), + Global: globalKcpInformers.Cache().V1alpha1().CachedResourceEndpointSlices().Informer(), + }, corev1alpha1.SchemeGroupVersion.WithResource("shards"): { Kind: "Shard", Local: localKcpInformers.Core().V1alpha1().Shards().Informer(), diff --git a/pkg/server/aggregatingcrdversiondiscovery/config.go b/pkg/server/aggregatingcrdversiondiscovery/config.go new file mode 100644 index 00000000000..769198511ae --- /dev/null +++ b/pkg/server/aggregatingcrdversiondiscovery/config.go @@ -0,0 +1,99 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aggregatingcrdversiondiscovery + +import ( + apiextensionsapiserverkcp "k8s.io/apiextensions-apiserver/pkg/kcp" + genericapiserver "k8s.io/apiserver/pkg/server" + + kcpapiextensionsv1informers "github.com/kcp-dev/client-go/apiextensions/informers/apiextensions/v1" + + apisv1alpha2informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha2" + corev1alpha1informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/core/v1alpha1" + apisv1alpha2listers "github.com/kcp-dev/kcp/sdk/client/listers/apis/v1alpha2" +) + +type Config struct { + Generic *genericapiserver.Config + Extra ExtraConfig +} + +type ExtraConfig struct { + CRDLister kcpapiextensionsv1informers.CustomResourceDefinitionClusterInformer + APIBindingAwareCRDLister apiextensionsapiserverkcp.ClusterAwareCRDClusterLister + APIBindingInformer apisv1alpha2informers.APIBindingClusterInformer + APIBindingLister apisv1alpha2listers.APIBindingClusterLister + LocalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer + GlobalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer + GlobalShardClusterInformer corev1alpha1informers.ShardClusterInformer +} + +type completedConfig struct { + Generic genericapiserver.CompletedConfig + Extra *ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig +} + +// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. +func (c *Config) Complete() CompletedConfig { + if c == nil { + return CompletedConfig{} + } + + cfg := completedConfig{ + c.Generic.Complete(nil), + &c.Extra, + } + // We do version discovery and nothing more. + cfg.Generic.SkipOpenAPIInstallation = true + + return CompletedConfig{&cfg} +} + +func (c *completedConfig) WithOpenAPIAggregationController(delegatedAPIServer *genericapiserver.GenericAPIServer) error { + return nil +} + +func NewConfig( + cfg *genericapiserver.Config, + crdLister kcpapiextensionsv1informers.CustomResourceDefinitionClusterInformer, + apiBindingAwareCRDLister apiextensionsapiserverkcp.ClusterAwareCRDClusterLister, + apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer, + localAPIExportInformer apisv1alpha2informers.APIExportClusterInformer, + globalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer, + globalShardClusterInformer corev1alpha1informers.ShardClusterInformer, +) (*Config, error) { + cfg.SkipOpenAPIInstallation = true + + ret := &Config{ + Generic: cfg, + Extra: ExtraConfig{ + CRDLister: crdLister, + APIBindingAwareCRDLister: apiBindingAwareCRDLister, + APIBindingInformer: apiBindingInformer, + LocalAPIExportInformer: localAPIExportInformer, + GlobalAPIExportInformer: globalAPIExportInformer, + GlobalShardClusterInformer: globalShardClusterInformer, + }, + } + + return ret, nil +} diff --git a/pkg/server/aggregatingcrdversiondiscovery/server.go b/pkg/server/aggregatingcrdversiondiscovery/server.go new file mode 100644 index 00000000000..1ecd24ef475 --- /dev/null +++ b/pkg/server/aggregatingcrdversiondiscovery/server.go @@ -0,0 +1,301 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aggregatingcrdversiondiscovery + +import ( + "fmt" + "net/http" + "strings" + + autoscaling "k8s.io/api/autoscaling/v1" + apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/endpoints/discovery" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/warning" + + "github.com/kcp-dev/logicalcluster/v3" + + "github.com/kcp-dev/kcp/pkg/indexers" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" +) + +var ( + scheme = runtime.NewScheme() + codecs = serializer.NewCodecFactory(scheme) + + unversionedVersion = schema.GroupVersion{Group: "", Version: "v1"} + unversionedTypes = []runtime.Object{ + &metav1.APIResourceList{}, + } +) + +func init() { + // we need to add the options to empty v1 + metav1.AddToGroupVersion(scheme, schema.GroupVersion{Group: "", Version: "v1"}) + + scheme.AddUnversionedTypes(unversionedVersion, unversionedTypes...) +} + +type Server struct { + GenericAPIServer *genericapiserver.GenericAPIServer + Extra *ExtraConfig + delegate http.Handler + + // Resources in /apis// may use storage other than CRD. + // Virtual resources are backed by virtual workspaces that may implement + // any verbs, and during discovery, we need to advertise correct set of + // verbs for each group-version tuple. storageAwareResourceVerbsProvider + // knows what storage is defined for which resource, and based on that + // information it can provide the correct set of verbs to advertise. + verbsProvider *storageAwareResourceVerbsProvider + + getCRD func(cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) + getAPIExportByPath func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) +} + +func NewServer(c CompletedConfig, delegationTarget genericapiserver.DelegationTarget) (*Server, error) { + s := &Server{ + Extra: c.Extra, + delegate: delegationTarget.UnprotectedHandler(), + + getCRD: func(clusterName logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { + return c.Extra.CRDLister.Lister().Cluster(clusterName).Get(name) + }, + getAPIExportByPath: func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport]( + apisv1alpha2.Resource("apiexports"), + c.Extra.LocalAPIExportInformer.Informer().GetIndexer(), + c.Extra.GlobalAPIExportInformer.Informer().GetIndexer(), + clusterPath, + name, + ) + }, + } + + s.verbsProvider = &storageAwareResourceVerbsProvider{ + getAPIExportByPath: s.getAPIExportByPath, + getAPIExportsByVirtualResourceIdentity: func(vrIdentity string) ([]*apisv1alpha2.APIExport, error) { + return indexers.ByIndexWithFallback[*apisv1alpha2.APIExport]( + c.Extra.LocalAPIExportInformer.Informer().GetIndexer(), + c.Extra.GlobalAPIExportInformer.Informer().GetIndexer(), + indexers.APIExportByVirtualResourceIdentities, + vrIdentity, + ) + }, + // For now we have only CachedResourceEndpointSlice as a source of virtual resources. + // The Replication VW supports only the verbs below. We just stash them here so + // that we don't have to do discovery each time we process a request. + knownVirtualResourceVerbs: map[string][]string{ + "CachedResourceEndpointSlice.cache.kcp.io": {"get", "list", "patch"}, + }, + knownVirtualResourceStatusVerbs: map[string][]string{ + "CachedResourceEndpointSlice.cache.kcp.io": {"get"}, + }, + knownVirtualResourceScaleVerbs: map[string][]string{ + "CachedResourceEndpointSlice.cache.kcp.io": {"get"}, + }, + } + + var err error + s.GenericAPIServer, err = c.Generic.New("aggregating-crd-version-discovery-apiserver", delegationTarget) + if err != nil { + return nil, err + } + + // We perform only APIResource discovery. + s.GenericAPIServer.DiscoveryGroupManager = nil + s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", s.newApisHandler()) + + return s, nil +} + +func splitPath(path string) []string { + path = strings.Trim(path, "/") + if path == "" { + return []string{} + } + return strings.Split(path, "/") +} + +func (s *Server) newApisHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if len(splitPath(r.URL.Path)) == 3 { + s.handleAPIResourceList(w, r) + return + } + + s.delegate.ServeHTTP(w, r) + } +} + +// apiResourcesForGroupVersion is taken from apiextensions-apiserver source, +// but we use the verbsProvider to get the list of supported verbs when +// constructing the APIResource objects. +func apiResourcesForGroupVersion(requestedGroup, requestedVersion string, crds []*apiextensionsv1.CustomResourceDefinition, verbsProvider resourceVerbsProvider) ([]metav1.APIResource, []error) { + apiResourcesForDiscovery := []metav1.APIResource{} + var errs []error + + for _, crd := range crds { + if requestedGroup != crd.Spec.Group { + continue + } + + if !apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.Established) { + continue + } + + var ( + storageVersionHash string + subresources *apiextensionsv1.CustomResourceSubresources + foundVersion = false + ) + + for _, v := range crd.Spec.Versions { + if !v.Served { + continue + } + + // HACK: support the case when we add core resources through CRDs (KCP scenario) + groupVersion := crd.Spec.Group + "/" + v.Name + if crd.Spec.Group == "" { + groupVersion = v.Name + } + + gv := metav1.GroupVersion{Group: groupVersion, Version: v.Name} + + if v.Name == requestedVersion { + foundVersion = true + subresources = v.Subresources + } + if v.Storage { + storageVersionHash = discovery.StorageVersionHash(logicalcluster.From(crd), gv.Group, gv.Version, crd.Spec.Names.Kind) + } + } + + if !foundVersion { + // This CRD doesn't have the requested version + continue + } + + resourceVerbs, err := verbsProvider.resource(crd) + if err != nil { + utilruntime.HandleError(err) + errs = append(errs, fmt.Errorf("%s.%s", crd.Status.AcceptedNames.Plural, crd.Spec.Group)) + continue + } + + apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ + Name: crd.Status.AcceptedNames.Plural, + SingularName: crd.Status.AcceptedNames.Singular, + Namespaced: crd.Spec.Scope == apiextensionsv1.NamespaceScoped, + Kind: crd.Status.AcceptedNames.Kind, + Verbs: resourceVerbs, + ShortNames: crd.Status.AcceptedNames.ShortNames, + Categories: crd.Status.AcceptedNames.Categories, + StorageVersionHash: storageVersionHash, + }) + + if subresources != nil && subresources.Status != nil { + statusVerbs, err := verbsProvider.statusSubresource(crd) + if err != nil { + utilruntime.HandleError(err) + errs = append(errs, fmt.Errorf("%s.%s status subresource", crd.Status.AcceptedNames.Plural, crd.Spec.Group)) + continue + } + + apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ + Name: crd.Status.AcceptedNames.Plural + "/status", + Namespaced: crd.Spec.Scope == apiextensionsv1.NamespaceScoped, + Kind: crd.Status.AcceptedNames.Kind, + Verbs: statusVerbs, + }) + } + + if subresources != nil && subresources.Scale != nil { + scaleVerbs, err := verbsProvider.scaleSubresource(crd) + if err != nil { + utilruntime.HandleError(err) + errs = append(errs, fmt.Errorf("%s.%s scale subresource", crd.Status.AcceptedNames.Plural, crd.Spec.Group)) + continue + } + + apiResourcesForDiscovery = append(apiResourcesForDiscovery, metav1.APIResource{ + Group: autoscaling.GroupName, + Version: "v1", + Kind: "Scale", + Name: crd.Status.AcceptedNames.Plural + "/scale", + Namespaced: crd.Spec.Scope == apiextensionsv1.NamespaceScoped, + Verbs: scaleVerbs, + }) + } + } + + return apiResourcesForDiscovery, errs +} + +func (s *Server) handleAPIResourceList(w http.ResponseWriter, r *http.Request) { + pathParts := splitPath(r.URL.Path) + // Only match /apis//. + if len(pathParts) != 3 || pathParts[0] != "apis" { + s.delegate.ServeHTTP(w, r) + return + } + + // We do only version discovery aggregation for CRDs. Reserved groups (apiextensions.kcp.io) don't belong here. + if strings.HasSuffix(pathParts[1], ".k8s.io") || strings.HasSuffix(pathParts[1], ".kubernetes.io") { + s.delegate.ServeHTTP(w, r) + return + } + + clusterName, wildcard, err := genericapirequest.ClusterNameOrWildcardFrom(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if wildcard { + // this is the only case where wildcard works for a list because this is our special CRD lister that handles it. + clusterName = "*" + } + + requestedGroup := pathParts[1] + requestedVersion := pathParts[2] + + crds, err := s.Extra.APIBindingAwareCRDLister.Cluster(clusterName).List(r.Context(), labels.Everything()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + apiResources, errs := apiResourcesForGroupVersion(requestedGroup, requestedVersion, crds, s.verbsProvider) + if len(errs) > 0 { + warning.AddWarning(r.Context(), "", fmt.Sprintf("Some resources are temporarily unavailable: %v.", errs)) + } + + resourceListerFunc := discovery.APIResourceListerFunc(func() []metav1.APIResource { + return apiResources + }) + + discovery.NewAPIVersionHandler(codecs, schema.GroupVersion{Group: requestedGroup, Version: requestedVersion}, resourceListerFunc).ServeHTTP(w, r) +} diff --git a/pkg/server/aggregatingcrdversiondiscovery/verbs.go b/pkg/server/aggregatingcrdversiondiscovery/verbs.go new file mode 100644 index 00000000000..af93538c5c5 --- /dev/null +++ b/pkg/server/aggregatingcrdversiondiscovery/verbs.go @@ -0,0 +1,170 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aggregatingcrdversiondiscovery + +import ( + "fmt" + "strings" + + apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/utils/ptr" + + "github.com/kcp-dev/logicalcluster/v3" + + apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" +) + +const ( + boundCRDVirtualStorageAnnotationPrefix = "virtual:" +) + +// resourceVerbsProvider provides verbs for a given virtual (sub)resource. +type resourceVerbsProvider interface { + resource(crd *apiextensionsv1.CustomResourceDefinition) (verbs []string, err error) + statusSubresource(crd *apiextensionsv1.CustomResourceDefinition) (verbs []string, err error) + scaleSubresource(crd *apiextensionsv1.CustomResourceDefinition) (verbs []string, err error) +} + +// storageAwareResourceVerbsProvider is able to list supported verbs of a resource based on its storage +// definition (.spec.resources[].storage) in its APIExport (if any), or fall-back to regular CRD verbs. +type storageAwareResourceVerbsProvider struct { + getAPIExportByPath func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) + getAPIExportsByVirtualResourceIdentity func(vrIdentity string) ([]*apisv1alpha2.APIExport, error) + + // These maps map . to the set of verbs to be + // advertised for that virtual resource. They are in a role of a cache: we know these verbs + // won't change, and so we don't have to do discovery against the backing virtual workspace + // each time we receive a request. + knownVirtualResourceVerbs map[string][]string + knownVirtualResourceStatusVerbs map[string][]string + knownVirtualResourceScaleVerbs map[string][]string +} + +func (p *storageAwareResourceVerbsProvider) getVirtualResourceStorage(apiExportIdentity, vrIdentity string, gr schema.GroupResource) (*apisv1alpha2.ResourceSchemaStorageVirtual, *apisv1alpha2.APIExport, error) { + apiExports, err := p.getAPIExportsByVirtualResourceIdentity(vrIdentity) + if err != nil { + return nil, nil, err + } + if len(apiExports) == 0 { + return nil, nil, fmt.Errorf("no APIExports for virtual resource identity %s", vrIdentity) + } + + var apiExport *apisv1alpha2.APIExport + for _, ae := range apiExports { + if ae.Status.IdentityHash == apiExportIdentity { + apiExport = ae + break + } + } + if apiExport == nil { + return nil, nil, fmt.Errorf("no matching APIExport for identity %s and virtual resource identity %s", apiExportIdentity, vrIdentity) + } + + var virtualStorage *apisv1alpha2.ResourceSchemaStorageVirtual + for _, resourceSchema := range apiExport.Spec.Resources { + if resourceSchema.Storage.Virtual != nil && + resourceSchema.Storage.Virtual.IdentityHash == vrIdentity && + resourceSchema.Group == gr.Group && + resourceSchema.Name == gr.Resource { + virtualStorage = resourceSchema.Storage.Virtual + break + } + } + + if virtualStorage == nil { + return nil, nil, fmt.Errorf("no APIExports for virtual resource %s with identity %s", gr, vrIdentity) + } + + return virtualStorage, apiExport, nil +} + +func (p *storageAwareResourceVerbsProvider) tryVirtualStorageVerbs(crd *apiextensionsv1.CustomResourceDefinition, verbsMap map[string][]string) ([]string, error) { + if crd.Annotations[apisv1alpha1.AnnotationSchemaStorageKey] != "" { + if !strings.HasPrefix(crd.Annotations[apisv1alpha1.AnnotationSchemaStorageKey], boundCRDVirtualStorageAnnotationPrefix) { + // We don't support any non-CRD storages other than virtual. + return nil, fmt.Errorf("unknown %s annotation %q on bound CRD %s", apisv1alpha1.AnnotationSchemaStorageKey, crd.Annotations[apisv1alpha1.AnnotationSchemaStorageKey], crd.Name) + } + + vrIdentity := strings.TrimPrefix(crd.Annotations[apisv1alpha1.AnnotationSchemaStorageKey], boundCRDVirtualStorageAnnotationPrefix) + virtualStorage, apiExport, err := p.getVirtualResourceStorage( + crd.Annotations[apisv1alpha1.AnnotationAPIIdentityKey], + vrIdentity, + schema.GroupResource{ + Group: crd.Spec.Group, + Resource: crd.Status.AcceptedNames.Plural, + }, + ) + if err != nil { + return nil, err + } + + // Check against known virtual resources. + if verbs, ok := verbsMap[fmt.Sprintf("%s.%s", virtualStorage.Reference.Kind, ptr.Deref(virtualStorage.Reference.APIGroup, "core"))]; ok { + return verbs, nil + } + + // TODO(gman0): add a fallback option to retrieve verbs for unknown/dynamically added VRs with real + // discovery from their VWs, if we ever need such a thing. For now we're just returning an error. + return nil, fmt.Errorf("unknown virtual resource endpoint slice %s.%s defined in %s|%s", virtualStorage.Reference.Kind, ptr.Deref(virtualStorage.Reference.APIGroup, "core"), logicalcluster.From(apiExport), apiExport.Name) + } + + return nil, nil +} + +func (p *storageAwareResourceVerbsProvider) resource(crd *apiextensionsv1.CustomResourceDefinition) ([]string, error) { + if virtualStorageVerbs, err := p.tryVirtualStorageVerbs(crd, p.knownVirtualResourceVerbs); err != nil { + return nil, err + } else if virtualStorageVerbs != nil { + return virtualStorageVerbs, nil + } + + // Resources with CRD storage get regular CRD verbs. + + verbs := metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}) + // if we're terminating we don't allow some verbs + if apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.Terminating) { + verbs = metav1.Verbs([]string{"delete", "deletecollection", "get", "list", "watch"}) + } + + return verbs, nil +} + +func (p *storageAwareResourceVerbsProvider) statusSubresource(crd *apiextensionsv1.CustomResourceDefinition) ([]string, error) { + if virtualStorageVerbs, err := p.tryVirtualStorageVerbs(crd, p.knownVirtualResourceStatusVerbs); err != nil { + return nil, err + } else if virtualStorageVerbs != nil { + return virtualStorageVerbs, nil + } + + // Resources with CRD storage get regular CRD status verbs. + return metav1.Verbs([]string{"get", "patch", "update"}), nil +} + +func (p *storageAwareResourceVerbsProvider) scaleSubresource(crd *apiextensionsv1.CustomResourceDefinition) ([]string, error) { + if virtualStorageVerbs, err := p.tryVirtualStorageVerbs(crd, p.knownVirtualResourceStatusVerbs); err != nil { + return nil, err + } else if virtualStorageVerbs != nil { + return virtualStorageVerbs, nil + } + + // Resources with CRD storage get regular CRD scale verbs. + return metav1.Verbs([]string{"get", "patch", "update"}), nil +} diff --git a/pkg/server/apiextensions.go b/pkg/server/apiextensions.go index 43ea6792c65..de1cd04f22e 100644 --- a/pkg/server/apiextensions.go +++ b/pkg/server/apiextensions.go @@ -36,6 +36,7 @@ import ( kcpapiextensionsv1listers "github.com/kcp-dev/client-go/apiextensions/listers/apiextensions/v1" "github.com/kcp-dev/logicalcluster/v3" + "github.com/kcp-dev/kcp/pkg/indexers" "github.com/kcp-dev/kcp/pkg/logging" "github.com/kcp-dev/kcp/pkg/reconciler/apis/apibinding" kcpfilters "github.com/kcp-dev/kcp/pkg/server/filters" @@ -63,14 +64,12 @@ type apiBindingAwareCRDClusterLister struct { apiBindingIndexer cache.Indexer apiBindingLister apisv1alpha2listers.APIBindingClusterLister - apiExportIndexer cache.Indexer - - getAPIResourceSchema func(clusterName logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) + getAPIExportByPath func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) } -func (a *apiBindingAwareCRDClusterLister) Cluster(name logicalcluster.Name) kcp.ClusterAwareCRDLister { +func (c *apiBindingAwareCRDClusterLister) Cluster(name logicalcluster.Name) kcp.ClusterAwareCRDLister { return &apiBindingAwareCRDLister{ - apiBindingAwareCRDClusterLister: a, + apiBindingAwareCRDClusterLister: c, cluster: name, } } @@ -85,6 +84,21 @@ type apiBindingAwareCRDLister struct { var _ kcp.ClusterAwareCRDLister = &apiBindingAwareCRDLister{} +func (c *apiBindingAwareCRDClusterLister) apiExportGetter(path logicalcluster.Path, name string) func() (*apisv1alpha2.APIExport, error) { + var apiExport *apisv1alpha2.APIExport + return func() (*apisv1alpha2.APIExport, error) { + if apiExport != nil { + return apiExport, nil + } + ae, err := c.getAPIExportByPath(path, name) + if err != nil { + return nil, err + } + apiExport = ae + return apiExport, nil + } +} + // List lists all CustomResourceDefinitions that come in via APIBindings as well as all in the current // logical cluster retrieved from the context. func (c *apiBindingAwareCRDLister) List(ctx context.Context, selector labels.Selector) ([]*apiextensionsv1.CustomResourceDefinition, error) { @@ -116,6 +130,12 @@ func (c *apiBindingAwareCRDLister) List(ctx context.Context, selector labels.Sel return nil, err } for _, apiBinding := range apiBindings { + apiExportPath := logicalcluster.NewPath(apiBinding.Spec.Reference.Export.Path) + if apiExportPath.Empty() { + apiExportPath = logicalcluster.NewPath(logicalcluster.From(apiBinding).String()) + } + apiExportGetter := c.apiExportGetter(apiExportPath, apiBinding.Spec.Reference.Export.Name) + for _, boundResource := range apiBinding.Status.BoundResources { logger := logging.WithObject(logger, &apiextensionsv1.CustomResourceDefinition{ ObjectMeta: metav1.ObjectMeta{ @@ -141,11 +161,22 @@ func (c *apiBindingAwareCRDLister) List(ctx context.Context, selector labels.Sel } // Priority 2: Add APIBinding CRDs. These take priority over those from the local workspace. - // Add the APIExport identity hash as an annotation to the CRD so the RESTOptionsGetter can assign // the correct etcd resource prefix. crd = decorateCRDWithBinding(crd, boundResource.Schema.IdentityHash, apiBinding.DeletionTimestamp) + if len(boundResource.StorageVersions) == 0 { + // No storage versions means this bound resource uses non-CRD storage. + // We'll need to consult with the source APIExport to see what it is, + // and optionally add storage annotation to inform the handlers to circumvent + // the regular apiextensions apiserver, and handle the resource accordingly. + crd, err = tryDecorateCRDWithSchemaStorage(crd, apiExportGetter) + if err != nil { + logger.Error(err, "skipping APIBinding CRD with non-CRD storage") + continue + } + } + ret = append(ret, crd) seen.Insert(crdName(crd)) } @@ -198,6 +229,11 @@ func (c *apiBindingAwareCRDLister) Refresh(crd *apiextensionsv1.CustomResourceDe refreshed.Annotations[apisv1alpha1.AnnotationAPIIdentityKey] = "placeholder" } + // Copy the storage annotation, if any. + if storageAnn := crd.Annotations[apisv1alpha1.AnnotationSchemaStorageKey]; storageAnn != "" { + refreshed.Annotations[apisv1alpha1.AnnotationSchemaStorageKey] = storageAnn + } + // If crd was only partial metadata, make sure refreshed is too if _, partialMetadata := crd.Annotations[annotationKeyPartialMetadata]; partialMetadata { addPartialMetadataCRDAnnotation(refreshed) @@ -233,13 +269,13 @@ func (c *apiBindingAwareCRDLister) Get(ctx context.Context, name string) (*apiex identity := kcpfilters.IdentityFromContext(ctx) if clusterName == "*" && identity != "" { // Priority 2: APIBinding CRD - crd, err = c.getForIdentityWildcard(name, identity) + crd, err = c.getForIdentityWildcard(ctx, name, identity) } else if clusterName == "*" && partialMetadataRequest { // Priority 3: partial metadata wildcard request crd, err = c.getForWildcardPartialMetadata(name) } else if clusterName != "*" { // Priority 4: normal CRD request - crd, err = c.get(clusterName, name, identity) + crd, err = c.get(ctx, clusterName, name, identity) } else { return nil, apierrors.NewNotFound(apiextensionsv1.Resource("customresourcedefinitions"), name) } @@ -299,6 +335,35 @@ func decorateCRDWithBinding(in *apiextensionsv1.CustomResourceDefinition, identi return out } +func tryDecorateCRDWithSchemaStorage(in *apiextensionsv1.CustomResourceDefinition, apiExportGetter func() (*apisv1alpha2.APIExport, error)) (*apiextensionsv1.CustomResourceDefinition, error) { + apiExport, err := apiExportGetter() + if err != nil { + return nil, err + } + + var ( + resourceStorage apisv1alpha2.ResourceSchemaStorage + foundResource bool + ) + for _, resource := range apiExport.Spec.Resources { + if resource.Group == in.Spec.Group && resource.Name == in.Status.AcceptedNames.Plural { + resourceStorage = resource.Storage + foundResource = true + break + } + } + if !foundResource { + return nil, fmt.Errorf("APIExport %s|%s does not export resource %s.%s", logicalcluster.From(apiExport), apiExport.Name, in.Status.AcceptedNames.Plural, in.Spec.Group) + } + out := shallowCopyCRDAndDeepCopyAnnotations(in) + + if resourceStorage.Virtual != nil { + out.Annotations[apisv1alpha1.AnnotationSchemaStorageKey] = fmt.Sprintf("virtual:%s", resourceStorage.Virtual.IdentityHash) + } + + return out, nil +} + // addPartialMetadataCRDAnnotation adds an annotation that marks this CRD as being // for a partial metadata request. func addPartialMetadataCRDAnnotation(crd *apiextensionsv1.CustomResourceDefinition) { @@ -308,12 +373,14 @@ func addPartialMetadataCRDAnnotation(crd *apiextensionsv1.CustomResourceDefiniti // getForIdentityWildcard handles finding the right CRD for an incoming wildcard request with identity, such as // // /clusters/*/apis/$group/$version/$resource:$identity. -func (c *apiBindingAwareCRDLister) getForIdentityWildcard(name, identity string) (*apiextensionsv1.CustomResourceDefinition, error) { +func (c *apiBindingAwareCRDLister) getForIdentityWildcard(ctx context.Context, name, identity string) (*apiextensionsv1.CustomResourceDefinition, error) { + logger := klog.FromContext(ctx) + group, resource := crdNameToGroupResource(name) - indexKey := identityGroupResourceKeyFunc(identity, group, resource) + indexKey := indexers.IdentityGroupResourceKeyFunc(identity, group, resource) - apiBindings, err := c.apiBindingIndexer.ByIndex(byIdentityGroupResource, indexKey) + apiBindings, err := c.apiBindingIndexer.ByIndex(indexers.APIBindingByIdentityAndGroupResource, indexKey) if err != nil { return nil, err } @@ -327,10 +394,17 @@ func (c *apiBindingAwareCRDLister) getForIdentityWildcard(name, identity string) apiBinding := apiBindings[0].(*apisv1alpha2.APIBinding) var boundCRDName string + // Bound resources with no storage versions don't use the CRD storage, + // and need to be annotated accordingly, to let the apiserver know how + // to handle the resource. + var needsStorageAnnotation bool for _, r := range apiBinding.Status.BoundResources { if r.Group == group && r.Resource == resource && r.Schema.IdentityHash == identity { boundCRDName = r.Schema.UID + if len(r.StorageVersions) == 0 { + needsStorageAnnotation = true + } break } } @@ -348,6 +422,18 @@ func (c *apiBindingAwareCRDLister) getForIdentityWildcard(name, identity string) // the correct etcd resource prefix. Use a shallow copy because deep copy is expensive (but deep copy the annotations). crd = decorateCRDWithBinding(crd, identity, apiBinding.DeletionTimestamp) + if needsStorageAnnotation { + apiExportPath := logicalcluster.NewPath(apiBinding.Spec.Reference.Export.Path) + if apiExportPath.Empty() { + apiExportPath = logicalcluster.NewPath(logicalcluster.From(apiBinding).String()) + } + crd, err = tryDecorateCRDWithSchemaStorage(crd, c.apiExportGetter(apiExportPath, apiBinding.Spec.Reference.Export.Name)) + if err != nil { + logger.Error(err, "cannot annotate CRD with storage information") + return nil, apierrors.NewServiceUnavailable(fmt.Sprintf("%s is currently unavailable", name)) + } + } + return crd, nil } @@ -370,7 +456,9 @@ func (c *apiBindingAwareCRDLister) getSystemCRD(_ logicalcluster.Name, name stri return c.crdLister.Cluster(SystemCRDClusterName).Get(name) } -func (c *apiBindingAwareCRDLister) get(clusterName logicalcluster.Name, name, identity string) (*apiextensionsv1.CustomResourceDefinition, error) { +func (c *apiBindingAwareCRDLister) get(ctx context.Context, clusterName logicalcluster.Name, name, identity string) (*apiextensionsv1.CustomResourceDefinition, error) { + logger := klog.FromContext(ctx) + var crd *apiextensionsv1.CustomResourceDefinition // Priority 1: see if it comes from any APIBindings @@ -381,6 +469,12 @@ func (c *apiBindingAwareCRDLister) get(clusterName logicalcluster.Name, name, id return nil, err } for _, apiBinding := range apiBindings { + apiExportPath := logicalcluster.NewPath(apiBinding.Spec.Reference.Export.Path) + if apiExportPath.Empty() { + apiExportPath = logicalcluster.NewPath(logicalcluster.From(apiBinding).String()) + } + apiExportGetter := c.apiExportGetter(apiExportPath, apiBinding.Spec.Reference.Export.Name) + for _, boundResource := range apiBinding.Status.BoundResources { // identity is empty string if the request is coming from a regular workspace client. // It is set if the request is coming from the virtual apiexport apiserver client. @@ -401,6 +495,14 @@ func (c *apiBindingAwareCRDLister) get(clusterName logicalcluster.Name, name, id // the correct etcd resource prefix. crd = decorateCRDWithBinding(crd, boundResource.Schema.IdentityHash, apiBinding.DeletionTimestamp) + if len(boundResource.StorageVersions) == 0 { + crd, err = tryDecorateCRDWithSchemaStorage(crd, apiExportGetter) + if err != nil { + logger.Error(err, "cannot annotate CRD with storage information") + return nil, apierrors.NewServiceUnavailable(fmt.Sprintf("%s is currently unavailable", name)) + } + } + return crd, nil } } diff --git a/pkg/server/config.go b/pkg/server/config.go index 5a680b43414..48384f67696 100644 --- a/pkg/server/config.go +++ b/pkg/server/config.go @@ -62,14 +62,17 @@ import ( "github.com/kcp-dev/kcp/pkg/authorization" bootstrappolicy "github.com/kcp-dev/kcp/pkg/authorization/bootstrap" kcpfeatures "github.com/kcp-dev/kcp/pkg/features" + "github.com/kcp-dev/kcp/pkg/indexers" "github.com/kcp-dev/kcp/pkg/informer" "github.com/kcp-dev/kcp/pkg/network" + "github.com/kcp-dev/kcp/pkg/server/aggregatingcrdversiondiscovery" "github.com/kcp-dev/kcp/pkg/server/bootstrap" kcpfilters "github.com/kcp-dev/kcp/pkg/server/filters" "github.com/kcp-dev/kcp/pkg/server/openapiv3" kcpserveroptions "github.com/kcp-dev/kcp/pkg/server/options" "github.com/kcp-dev/kcp/pkg/server/options/batteries" - apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + "github.com/kcp-dev/kcp/pkg/server/virtualresources" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" @@ -81,11 +84,13 @@ type Config struct { EmbeddedEtcd *embeddedetcd.Config - GenericConfig *genericapiserver.Config // the config embedded into MiniAggregator, the head of the delegation chain - MiniAggregator *miniaggregator.MiniAggregatorConfig - Apis *controlplaneapiserver.Config - ApiExtensions *apiextensionsapiserver.Config - OptionalVirtual *VirtualConfig + GenericConfig *genericapiserver.Config // the config embedded into MiniAggregator, the head of the delegation chain + MiniAggregator *miniaggregator.MiniAggregatorConfig + Apis *controlplaneapiserver.Config + ApiExtensions *apiextensionsapiserver.Config + VirtualResources *virtualresources.Config + AggregatingCRDVersionDiscovery *aggregatingcrdversiondiscovery.Config + OptionalVirtual *VirtualConfig ExtraConfig } @@ -141,12 +146,14 @@ type ExtraConfig struct { type completedConfig struct { Options kcpserveroptions.CompletedOptions - GenericConfig genericapiserver.CompletedConfig - EmbeddedEtcd embeddedetcd.CompletedConfig - MiniAggregator miniaggregator.CompletedMiniAggregatorConfig - Apis controlplaneapiserver.CompletedConfig - ApiExtensions apiextensionsapiserver.CompletedConfig - OptionalVirtual CompletedVirtualConfig + GenericConfig genericapiserver.CompletedConfig + EmbeddedEtcd embeddedetcd.CompletedConfig + MiniAggregator miniaggregator.CompletedMiniAggregatorConfig + Apis controlplaneapiserver.CompletedConfig + ApiExtensions apiextensionsapiserver.CompletedConfig + VirtualResources virtualresources.CompletedConfig + AggregatingCRDVersionDiscovery aggregatingcrdversiondiscovery.CompletedConfig + OptionalVirtual CompletedVirtualConfig ExtraConfig } @@ -163,11 +170,13 @@ func (c *Config) Complete() (CompletedConfig, error) { return CompletedConfig{&completedConfig{ Options: c.Options, - GenericConfig: c.GenericConfig.Complete(informerfactoryhack.Wrap(c.KubeSharedInformerFactory)), - EmbeddedEtcd: c.EmbeddedEtcd.Complete(), - MiniAggregator: miniAggregator, - Apis: c.Apis.Complete(), - ApiExtensions: c.ApiExtensions.Complete(), + GenericConfig: c.GenericConfig.Complete(informerfactoryhack.Wrap(c.KubeSharedInformerFactory)), + EmbeddedEtcd: c.EmbeddedEtcd.Complete(), + MiniAggregator: miniAggregator, + Apis: c.Apis.Complete(), + ApiExtensions: c.ApiExtensions.Complete(), + AggregatingCRDVersionDiscovery: c.AggregatingCRDVersionDiscovery.Complete(), + VirtualResources: c.VirtualResources.Complete(), OptionalVirtual: c.OptionalVirtual.Complete( miniAggregator.GenericConfig.Authentication, miniAggregator.GenericConfig.AuditPolicyRuleEvaluator, @@ -383,6 +392,7 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co } var virtualWorkspaceServerProxyTransport http.RoundTripper + var virtualWorkspaceTranposportTLSClientConfig *tls.Config if opts.Extra.ShardClientCertFile != "" && opts.Extra.ShardClientKeyFile != "" && opts.Extra.ShardVirtualWorkspaceCAFile != "" { caCert, err := os.ReadFile(opts.Extra.ShardVirtualWorkspaceCAFile) if err != nil { @@ -398,10 +408,11 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co } transport := http.DefaultTransport.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{ + virtualWorkspaceTranposportTLSClientConfig = &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caCertPool, } + transport.TLSClientConfig = virtualWorkspaceTranposportTLSClientConfig virtualWorkspaceServerProxyTransport = transport } @@ -613,24 +624,89 @@ func NewConfig(ctx context.Context, opts kcpserveroptions.CompletedOptions) (*Co _ = c.KcpSharedInformerFactory.Apis().V1alpha1().APIConversions().Informer() _ = c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Informer().GetIndexer().AddIndexers(cache.Indexers{byGroupResourceName: indexCRDByGroupResourceName}) - _ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer().AddIndexers(cache.Indexers{byIdentityGroupResource: indexAPIBindingByIdentityGroupResource}) + _ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer().AddIndexers(cache.Indexers{ + indexers.APIBindingByIdentityAndGroupResource: indexers.IndexAPIBindingByIdentityGroupResource, + indexers.APIBindingByBoundResources: indexers.IndexAPIBindingByBoundResources, + }) + _ = c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer().AddIndexers(cache.Indexers{ + indexers.APIExportByVirtualResourceIdentities: indexers.IndexAPIExportByVirtualResourceIdentities, + }) + _ = c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer().AddIndexers(cache.Indexers{ + indexers.APIExportByVirtualResourceIdentities: indexers.IndexAPIExportByVirtualResourceIdentities, + }) - c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister = &apiBindingAwareCRDClusterLister{ + apiBindingAwareCRDClusterLister := &apiBindingAwareCRDClusterLister{ kcpClusterClient: c.KcpClusterClient, crdLister: c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Lister(), crdIndexer: c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions().Informer().GetIndexer(), workspaceLister: c.KcpSharedInformerFactory.Tenancy().V1alpha1().Workspaces().Lister(), apiBindingLister: c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Lister(), apiBindingIndexer: c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().GetIndexer(), - apiExportIndexer: c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(), - getAPIResourceSchema: func(clusterName logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) { - return c.KcpSharedInformerFactory.Apis().V1alpha1().APIResourceSchemas().Lister().Cluster(clusterName).Get(name) + getAPIExportByPath: func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport]( + apisv1alpha2.Resource("apiexports"), + c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + clusterPath, + name, + ) }, } + c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister = apiBindingAwareCRDClusterLister c.ApiExtensions.ExtraConfig.Client = c.ApiExtensionsClusterClient c.ApiExtensions.ExtraConfig.Informers = c.ApiExtensionsSharedInformerFactory c.ApiExtensions.ExtraConfig.TableConverterProvider = NewTableConverterProvider() + if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.CacheAPIs) { + // We need an aggregating version discovery for CRDs that is RESTstorage-aware. + // The apiextensions apiserver sources its data from the apiBindingAwareCRDClusterLister. + // With the onset of virtual resources, not all bound CRDs use CRD storage, and as a consequence, + // the verbs such a resource supports may also be different -- compared to what apiextensions + // thinks a CRD should support. This special version discovery server is aware of the storage + // defined in the APIExport of the bound CRD, and will advertise correct set of verbs in + // APIResourceList for CRD- or virtual-backed resources. + aggregatingVersionDiscoveryConfig := *c.GenericConfig + c.AggregatingCRDVersionDiscovery, err = aggregatingcrdversiondiscovery.NewConfig( + &aggregatingVersionDiscoveryConfig, + c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(), + apiBindingAwareCRDClusterLister, + c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(), + c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + c.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards(), + ) + if err != nil { + return nil, fmt.Errorf("failed to create config for aggregating version discovery server: %v", err) + } + + // The virtual resources apiserver serves resources from a virtual workspace. + virtualResourcesConfig := *c.GenericConfig + virtualResourcesConfig.SkipOpenAPIInstallation = true + // vwClientConfig is used by the proxy handler to proxy the client requests to the virtual workspace. + vwClientConfig := rest.CopyConfig(c.GenericConfig.LoopbackClientConfig) + if !opts.Virtual.Enabled { + vwClientConfig.TLSClientConfig = rest.TLSClientConfig{ + CAFile: opts.Extra.ShardVirtualWorkspaceCAFile, + CertFile: opts.Extra.ShardClientCertFile, + KeyFile: opts.Extra.ShardClientKeyFile, + } + } + + c.VirtualResources, err = virtualresources.NewConfig( + &virtualResourcesConfig, + vwClientConfig, + c.CacheDynamicClient, + c.ShardVirtualWorkspaceURL, + c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions(), + c.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(), + c.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + c.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + ) + if err != nil { + return nil, fmt.Errorf("failed to create config for virtual resources server: %v", err) + } + } + c.openAPIv3Controller = openapiv3.NewController(c.ApiExtensionsSharedInformerFactory.Apiextensions().V1().CustomResourceDefinitions()) c.openAPIv3ServiceCache = openapiv3.NewServiceCache(c.GenericConfig.OpenAPIV3Config, c.ApiExtensions.ExtraConfig.ClusterAwareCRDLister, c.openAPIv3Controller, openapiv3.DefaultServiceCacheSize) diff --git a/pkg/server/controllers.go b/pkg/server/controllers.go index 2c8c34e81a4..6774d66cb9d 100644 --- a/pkg/server/controllers.go +++ b/pkg/server/controllers.go @@ -72,6 +72,7 @@ import ( apisreplicateclusterrolebinding "github.com/kcp-dev/kcp/pkg/reconciler/apis/replicateclusterrolebinding" apisreplicatelogicalcluster "github.com/kcp-dev/kcp/pkg/reconciler/apis/replicatelogicalcluster" "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresourceendpointslice" + "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresourceendpointsliceurls" "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresources" "github.com/kcp-dev/kcp/pkg/reconciler/cache/labelclusterrolebindings" "github.com/kcp-dev/kcp/pkg/reconciler/cache/labelclusterroles" @@ -1778,31 +1779,71 @@ func (s *Server) installCachedResourceEndpointSliceController(ctx context.Contex if err != nil { return err } - cachedResourceEndpointSliceInformer := s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices() - cachedResourceInformer := s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResources() - lcClusterInformer := s.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters() - apiBindingClusterInfomer := s.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings() - c, err := cachedresourceendpointslice.NewController( + s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices(), + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources(), + s.KcpSharedInformerFactory.Topology().V1alpha1().Partitions(), + kcpClusterClient, + ) + if err != nil { + return err + } + return s.registerController(&controllerWrapper{ + Name: cachedresourceendpointslice.ControllerName, + Wait: func(ctx context.Context, s *Server) error { + return wait.PollUntilContextCancel(ctx, waitPollInterval, true, func(ctx context.Context) (bool, error) { + return s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices().Informer().HasSynced() && + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources().Informer().HasSynced() && + s.KcpSharedInformerFactory.Topology().V1alpha1().Partitions().Informer().HasSynced(), nil + }) + }, + Runner: func(ctx context.Context) { + c.Start(ctx, 2) + }, + }) +} + +func (s *Server) installCachedResourceEndpointSliceURLsController(_ context.Context, config *rest.Config) error { + if !kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.CacheAPIs) { + return nil + } + + config = rest.CopyConfig(config) + config = rest.AddUserAgent(config, cachedresourceendpointsliceurls.ControllerName) + + kcpClusterClient, err := kcpclientset.NewForConfig(config) + if err != nil { + return err + } + + c, err := cachedresourceendpointsliceurls.NewController( s.Options.Extra.ShardName, - cachedResourceEndpointSliceInformer, - cachedResourceInformer, + s.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings(), + s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices(), + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices(), s.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards(), - lcClusterInformer, - apiBindingClusterInfomer, + s.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + s.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources(), + s.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters(), kcpClusterClient, ) if err != nil { return err } + return s.registerController(&controllerWrapper{ - Name: cachedresourceendpointslice.ControllerName, + Name: cachedresourceendpointsliceurls.ControllerName, Wait: func(ctx context.Context, s *Server) error { return wait.PollUntilContextCancel(ctx, waitPollInterval, true, func(ctx context.Context) (bool, error) { - return cachedResourceEndpointSliceInformer.Informer().HasSynced() && - cachedResourceInformer.Informer().HasSynced() && - lcClusterInformer.Informer().HasSynced() && - apiBindingClusterInfomer.Informer().HasSynced(), nil + return s.KcpSharedInformerFactory.Apis().V1alpha2().APIBindings().Informer().HasSynced() && + s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices().Informer().HasSynced() && + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices().Informer().HasSynced() && + s.CacheKcpSharedInformerFactory.Core().V1alpha1().Shards().Informer().HasSynced() && + s.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().HasSynced() && + s.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().HasSynced() && + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources().Informer().HasSynced() && + s.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters().Informer().HasSynced(), nil }) }, Runner: func(ctx context.Context) { @@ -1878,7 +1919,16 @@ func (s *Server) addIndexersToInformers(_ context.Context) map[schema.GroupVersi s.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), ) cachedresourceendpointslice.InstallIndexers( + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources(), + s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices(), + ) + cachedresourceendpointsliceurls.InstallIndexers( + s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResources(), + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources(), s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices(), + s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices(), + s.KcpSharedInformerFactory.Apis().V1alpha2().APIExports(), + s.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports(), ) return replication.InstallIndexers( s.KcpSharedInformerFactory, diff --git a/pkg/server/indexes.go b/pkg/server/indexes.go index 533451e389a..2d20a4bd68c 100644 --- a/pkg/server/indexes.go +++ b/pkg/server/indexes.go @@ -20,13 +20,10 @@ import ( "fmt" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - - apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" ) const ( - byGroupResourceName = "byGroupResourceName" // ., core group uses "core" - byIdentityGroupResource = "byIdentityGroupResource" + byGroupResourceName = "byGroupResourceName" // ., core group uses "core" ) func indexCRDByGroupResourceName(obj interface{}) ([]string, error) { @@ -41,22 +38,3 @@ func indexCRDByGroupResourceName(obj interface{}) ([]string, error) { } return []string{fmt.Sprintf("%s.%s", crd.Spec.Names.Plural, group)}, nil } - -func indexAPIBindingByIdentityGroupResource(obj interface{}) ([]string, error) { - apiBinding, ok := obj.(*apisv1alpha2.APIBinding) - if !ok { - return []string{}, fmt.Errorf("obj is supposed to be an APIBinding, but is %T", obj) - } - - ret := make([]string, 0, len(apiBinding.Status.BoundResources)) - - for _, r := range apiBinding.Status.BoundResources { - ret = append(ret, identityGroupResourceKeyFunc(r.Schema.IdentityHash, r.Group, r.Resource)) - } - - return ret, nil -} - -func identityGroupResourceKeyFunc(identity, group, resource string) string { - return fmt.Sprintf("%s/%s/%s", identity, group, resource) -} diff --git a/pkg/server/server.go b/pkg/server/server.go index 0a74e093daf..9aa0916da63 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -53,12 +53,15 @@ import ( configshard "github.com/kcp-dev/kcp/config/shard" systemcrds "github.com/kcp-dev/kcp/config/system-crds" bootstrappolicy "github.com/kcp-dev/kcp/pkg/authorization/bootstrap" + kcpfeatures "github.com/kcp-dev/kcp/pkg/features" "github.com/kcp-dev/kcp/pkg/informer" metadataclient "github.com/kcp-dev/kcp/pkg/metadata" "github.com/kcp-dev/kcp/pkg/reconciler/cache/replication" "github.com/kcp-dev/kcp/pkg/reconciler/dynamicrestmapper" "github.com/kcp-dev/kcp/pkg/reconciler/kubequota" + "github.com/kcp-dev/kcp/pkg/server/aggregatingcrdversiondiscovery" "github.com/kcp-dev/kcp/pkg/server/options/batteries" + "github.com/kcp-dev/kcp/pkg/server/virtualresources" virtualrootapiserver "github.com/kcp-dev/kcp/pkg/virtual/framework/rootapiserver" "github.com/kcp-dev/kcp/sdk/apis/core" corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" @@ -71,10 +74,12 @@ const resyncPeriod = 10 * time.Hour type Server struct { CompletedConfig - ApiExtensions *extensionsapiserver.CustomResourceDefinitions - Apis *controlplaneapiserver.Server - MiniAggregator *miniaggregator.MiniAggregatorServer - virtual *virtualrootapiserver.Server + ApiExtensions *extensionsapiserver.CustomResourceDefinitions + Apis *controlplaneapiserver.Server + VirtualResources *virtualresources.Server + AggregatingCRDVersionDiscovery *aggregatingcrdversiondiscovery.Server + MiniAggregator *miniaggregator.MiniAggregatorServer + virtual *virtualrootapiserver.Server // DynRESTMapper is a workspace-aware REST mapper, backed by a reconciler, // which dynamically loads all bound resources through every type associated // with an APIBinding in the workspace into the mapper. Another controller can @@ -111,7 +116,21 @@ func NewServer(c CompletedConfig) (*Server, error) { return nil, fmt.Errorf("create api extensions: %v", err) } - s.Apis, err = c.Apis.New("generic-control-plane", s.ApiExtensions.GenericAPIServer) + gcpDelegate := s.ApiExtensions.GenericAPIServer + + if kcpfeatures.DefaultFeatureGate.Enabled(kcpfeatures.CacheAPIs) { + s.VirtualResources, err = virtualresources.NewServer(c.VirtualResources, s.ApiExtensions.GenericAPIServer, s.DynRESTMapper) + if err != nil { + return nil, fmt.Errorf("failed to create virtual resources server: %v", err) + } + s.AggregatingCRDVersionDiscovery, err = aggregatingcrdversiondiscovery.NewServer(c.AggregatingCRDVersionDiscovery, s.VirtualResources.GenericAPIServer) + if err != nil { + return nil, fmt.Errorf("failed to create aggregating version discovery server: %v", err) + } + gcpDelegate = s.AggregatingCRDVersionDiscovery.GenericAPIServer + } + + s.Apis, err = c.Apis.New("generic-control-plane", gcpDelegate) if err != nil { return nil, fmt.Errorf("failed to create generic controlplane apiserver: %w", err) } @@ -378,6 +397,9 @@ func (s *Server) installControllers(ctx context.Context, controllerConfig *rest. if err := s.installCachedResourceEndpointSliceController(ctx, controllerConfig); err != nil { return err } + if err := s.installCachedResourceEndpointSliceURLsController(ctx, s.ExternalLogicalClusterAdminConfig); err != nil { + return err + } } return nil @@ -451,8 +473,11 @@ func (s *Server) Run(ctx context.Context) error { go s.KcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().Run(hookContext.Done()) go s.KcpSharedInformerFactory.Apis().V1alpha1().APIExportEndpointSlices().Informer().Run(hookContext.Done()) go s.CacheKcpSharedInformerFactory.Apis().V1alpha2().APIExports().Informer().Run(hookContext.Done()) + go s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResources().Informer().Run(hookContext.Done()) + go s.CacheKcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices().Informer().Run(hookContext.Done()) go s.KcpSharedInformerFactory.Core().V1alpha1().LogicalClusters().Informer().Run(hookContext.Done()) go s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResources().Informer().Run(hookContext.Done()) + go s.KcpSharedInformerFactory.Cache().V1alpha1().CachedResourceEndpointSlices().Informer().Run(hookContext.Done()) logger.Info("starting APIExport, APIBinding and LogicalCluster informers") if err := wait.PollUntilContextCancel(hookCtx, time.Millisecond*100, true, func(ctx context.Context) (bool, error) { diff --git a/pkg/server/virtualresources/config.go b/pkg/server/virtualresources/config.go new file mode 100644 index 00000000000..21107a526f3 --- /dev/null +++ b/pkg/server/virtualresources/config.go @@ -0,0 +1,113 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualresources + +import ( + "k8s.io/apimachinery/pkg/runtime" + apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/client-go/rest" + openapicommon "k8s.io/kube-openapi/pkg/common" + + kcpapiextensionsv1informers "github.com/kcp-dev/client-go/apiextensions/informers/apiextensions/v1" + kcpdynamic "github.com/kcp-dev/client-go/dynamic" + + apisv1alpha2informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha2" +) + +type Config struct { + Generic *genericapiserver.Config + Extra ExtraConfig +} + +type ExtraConfig struct { + VWClientConfig *rest.Config + DynamicClusterClient kcpdynamic.ClusterInterface + + ShardVirtualWorkspaceURLGetter func() string + + CRDLister kcpapiextensionsv1informers.CustomResourceDefinitionClusterInformer + APIBindingInformer apisv1alpha2informers.APIBindingClusterInformer + LocalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer + GlobalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer +} + +type completedConfig struct { + Generic genericapiserver.CompletedConfig + Extra *ExtraConfig +} + +type CompletedConfig struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedConfig +} + +// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver. +func (c *Config) Complete() CompletedConfig { + if c == nil { + return CompletedConfig{} + } + + cfg := completedConfig{ + c.Generic.Complete(nil), + &c.Extra, + } + + cfg.Generic.OpenAPIV3Config = genericapiserver.DefaultOpenAPIV3Config( + func(rc openapicommon.ReferenceCallback) map[string]openapicommon.OpenAPIDefinition { + return map[string]openapicommon.OpenAPIDefinition{} + }, + apiopenapi.NewDefinitionNamer(runtime.NewScheme()), + ) + + return CompletedConfig{&cfg} +} + +func (c *completedConfig) WithOpenAPIAggregationController(delegatedAPIServer *genericapiserver.GenericAPIServer) error { + return nil +} + +func NewConfig( + cfg *genericapiserver.Config, + vwClientConfig *rest.Config, + dynamicClusterClient kcpdynamic.ClusterInterface, + shardVirtualWorkspaceURLGetter func() string, + crdLister kcpapiextensionsv1informers.CustomResourceDefinitionClusterInformer, + apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer, + localAPIExportInformer apisv1alpha2informers.APIExportClusterInformer, + globalAPIExportInformer apisv1alpha2informers.APIExportClusterInformer, +) (*Config, error) { + rest.AddUserAgent(vwClientConfig, "kcp-virtual-resources-apiserver") + cfg.SkipOpenAPIInstallation = true + + ret := &Config{ + Generic: cfg, + Extra: ExtraConfig{ + VWClientConfig: vwClientConfig, + DynamicClusterClient: dynamicClusterClient, + + ShardVirtualWorkspaceURLGetter: shardVirtualWorkspaceURLGetter, + + CRDLister: crdLister, + APIBindingInformer: apiBindingInformer, + LocalAPIExportInformer: localAPIExportInformer, + GlobalAPIExportInformer: globalAPIExportInformer, + }, + } + + return ret, nil +} diff --git a/pkg/server/virtualresources/server.go b/pkg/server/virtualresources/server.go new file mode 100644 index 00000000000..5f5feb6f05d --- /dev/null +++ b/pkg/server/virtualresources/server.go @@ -0,0 +1,395 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package virtualresources + +import ( + "context" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + "github.com/kcp-dev/logicalcluster/v3" + + "github.com/kcp-dev/kcp/pkg/endpointslice" + "github.com/kcp-dev/kcp/pkg/indexers" + "github.com/kcp-dev/kcp/pkg/reconciler/dynamicrestmapper" + kcpfilters "github.com/kcp-dev/kcp/pkg/server/filters" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" +) + +var ( + errorScheme = runtime.NewScheme() + errorCodecs = serializer.NewCodecFactory(errorScheme) +) + +func init() { + errorScheme.AddUnversionedTypes(metav1.Unversioned, + &metav1.Status{}, + ) +} + +type Server struct { + GenericAPIServer *genericapiserver.GenericAPIServer + Extra *ExtraConfig + drm *dynamicrestmapper.DynamicRESTMapper + delegate http.Handler + + getCRD func(cluster logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) + getUnstructuredEndpointSlice func(ctx context.Context, cluster logicalcluster.Name, gvr schema.GroupVersionResource, name string) (*unstructured.Unstructured, error) + getAPIExportByPath func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) +} + +func NewServer(c CompletedConfig, delegationTarget genericapiserver.DelegationTarget, drm *dynamicrestmapper.DynamicRESTMapper) (*Server, error) { + s := &Server{ + drm: drm, + Extra: c.Extra, + delegate: delegationTarget.UnprotectedHandler(), + + getUnstructuredEndpointSlice: func(ctx context.Context, cluster logicalcluster.Name, gvr schema.GroupVersionResource, name string) (*unstructured.Unstructured, error) { + list, err := c.Extra.DynamicClusterClient.Cluster(cluster.Path()).Resource(gvr).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + if len(list.Items) == 0 { + return nil, apierrors.NewNotFound(gvr.GroupResource(), name) + } + + var slice *unstructured.Unstructured + for _, item := range list.Items { + if item.GetName() == name { + if slice != nil { + return nil, apierrors.NewInternalError(fmt.Errorf("multiple objects found")) + } + slice = &item + } + } + + return slice, nil + }, + getCRD: func(clusterName logicalcluster.Name, name string) (*apiextensionsv1.CustomResourceDefinition, error) { + return c.Extra.CRDLister.Lister().Cluster(clusterName).Get(name) + }, + getAPIExportByPath: func(clusterPath logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport]( + apisv1alpha2.Resource("apiexports"), + c.Extra.LocalAPIExportInformer.Informer().GetIndexer(), + c.Extra.GlobalAPIExportInformer.Informer().GetIndexer(), + clusterPath, + name, + ) + }, + } + + var err error + s.GenericAPIServer, err = c.Generic.New("virtual-resources-root-apiserver", delegationTarget) + if err != nil { + return nil, err + } + + // We don't do discovery at all because it needs to be aggregated with other CRD-based resources. + s.GenericAPIServer.DiscoveryGroupManager = nil + s.GenericAPIServer.Handler.NonGoRestfulMux.HandlePrefix("/apis/", s.newApisHandler()) + + return s, nil +} + +func splitPath(path string) []string { + path = strings.Trim(path, "/") + if path == "" { + return []string{} + } + return strings.Split(path, "/") +} + +func (s *Server) newApisHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if len(splitPath(r.URL.Path)) > 3 { + s.handleResource(w, r) + return + } + + s.delegate.ServeHTTP(w, r) + } +} + +func (s *Server) handleResource(w http.ResponseWriter, r *http.Request) { + pathParts := splitPath(r.URL.Path) + // Only match /apis////... + if len(pathParts) <= 3 || pathParts[0] != "apis" { + s.delegate.ServeHTTP(w, r) + return + } + + ctx := r.Context() + requestInfo, ok := genericapirequest.RequestInfoFrom(ctx) + if !ok { + responsewriters.ErrorNegotiated( + apierrors.NewInternalError(fmt.Errorf("no RequestInfo found in the context")), + errorCodecs, schema.GroupVersion{}, w, r, + ) + return + } + if !requestInfo.IsResourceRequest { + // Discovery requests should have been caught earlier. + // Maybe the delegate knows what to do. + s.delegate.ServeHTTP(w, r) + return + } + + clusterNameOrWildcard, wildcard, err := genericapirequest.ClusterNameOrWildcardFrom(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if wildcard { + clusterNameOrWildcard = "*" + } + + gr := schema.GroupResource{ + Group: requestInfo.APIGroup, + Resource: requestInfo.Resource, + } + if gr.Group == "" { + gr.Group = "core" + } + + // partialMetadataRequest := kcpfilters.IsPartialMetadataRequest(ctx) + identity := kcpfilters.IdentityFromContext(ctx) + + apiBinding, err := s.getAPIBindingForRequest(clusterNameOrWildcard.String(), gr, identity) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if apiBinding == nil { + // Not a virtual resource: the resource is not provided by an APIBinding. + s.delegate.ServeHTTP(w, r) + return + } + + var crdName string + for _, boundResource := range apiBinding.Status.BoundResources { + if boundResource.Group == gr.Group && boundResource.Resource == gr.Resource { + crdName = boundResource.Schema.UID + + if len(boundResource.StorageVersions) > 0 { + // Virtual resources have zero storage versions, because they don't + // use CRD storage. This resource is definitely not a VR. + s.delegate.ServeHTTP(w, r) + return + } + + break + } + } + if crdName == "" { + // This should not happen, the indexers returned a binding for this specific GR. + responsewriters.ErrorNegotiated( + apierrors.NewInternalError(fmt.Errorf("resource not available")), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + + // We do what the apiextensions apiserver does: return 404 on not found, !NamesAccepted or !Established. + crd, err := s.getCRD(logicalcluster.Name("system:bound-crds"), crdName) + if err != nil { + if apierrors.IsNotFound(err) { + responsewriters.ErrorNegotiated( + apierrors.NewNotFound(schema.GroupResource{Group: requestInfo.APIGroup, Resource: requestInfo.Resource}, requestInfo.Name), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + utilruntime.HandleError(err) + responsewriters.ErrorNegotiated( + apierrors.NewInternalError(fmt.Errorf("error resolving resource: %v", err)), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + if !apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.NamesAccepted) && + !apiextensionshelpers.IsCRDConditionTrue(crd, apiextensionsv1.Established) { + responsewriters.ErrorNegotiated( + apierrors.NewNotFound(schema.GroupResource{Group: requestInfo.APIGroup, Resource: requestInfo.Resource}, requestInfo.Name), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + + // Get the origin APIExport, and check that the resource has virtual storage. Otherwise delegate the request. + + apiExportPath := logicalcluster.NewPath(apiBinding.Spec.Reference.Export.Path) + if apiExportPath.Empty() { + apiExportPath = logicalcluster.NewPath(logicalcluster.From(apiBinding).String()) + } + apiExport, err := s.getAPIExportByPath(apiExportPath, apiBinding.Spec.Reference.Export.Name) + if err != nil { + utilruntime.HandleError(err) + responsewriters.ErrorNegotiated( + apierrors.NewInternalError(fmt.Errorf("error resolving resource: %v", err)), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + + var virtualStorage *apisv1alpha2.ResourceSchemaStorageVirtual + for _, resource := range apiExport.Spec.Resources { + if resource.Storage.Virtual != nil && + resource.Group == gr.Group && + resource.Name == gr.Resource { + virtualStorage = resource.Storage.Virtual + break + } + } + if virtualStorage == nil { + // Not a virtual resource: the binding's export doesn't define such resource with virtual storage. + s.delegate.ServeHTTP(w, r) + return + } + + // We have a virtual resource. Get the endpoint URL, create a proxy handler and serve from that endpoint. + + vrEndpointURL, err := s.getVirtualResourceURL(ctx, logicalcluster.From(apiExport), virtualStorage) + if err != nil { + utilruntime.HandleError(err) + responsewriters.ErrorNegotiated( + apierrors.NewInternalError(fmt.Errorf("error resolving resource: %v", err)), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + + vrHandler, err := newVirtualResourceHandler(s.Extra.VWClientConfig, vrEndpointURL, apiExport.Status.IdentityHash, clusterNameOrWildcard.String()) + if err != nil { + utilruntime.HandleError(err) + responsewriters.ErrorNegotiated( + apierrors.NewInternalError(fmt.Errorf("error serving resource: %v", err)), + errorCodecs, schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}, w, r, + ) + return + } + + vrHandler.ServeHTTP(w, r) +} + +func (s *Server) getVirtualResourceURL(ctx context.Context, apiExportCluster logicalcluster.Name, virtual *apisv1alpha2.ResourceSchemaStorageVirtual) (string, error) { + sliceMapping, err := s.drm.ForCluster(apiExportCluster).RESTMapping(schema.GroupKind{ + Group: ptr.Deref(virtual.Reference.APIGroup, ""), + Kind: virtual.Reference.Kind, + }) + if err != nil { + return "", err + } + + slice, err := s.getUnstructuredEndpointSlice(ctx, apiExportCluster, schema.GroupVersionResource{ + Group: sliceMapping.Resource.Group, + Version: sliceMapping.Resource.Version, + Resource: sliceMapping.Resource.Resource, + }, virtual.Reference.Name) + if err != nil { + return "", err + } + + urls, err := endpointslice.ListURLsFromUnstructured(*slice) + if err != nil { + return "", err + } + + return endpointslice.FindOneURL(s.Extra.ShardVirtualWorkspaceURLGetter(), urls) +} + +func (s *Server) getAPIBindingForRequest( + clusterNameOrWildcard string, + gr schema.GroupResource, + identity string, +) (*apisv1alpha2.APIBinding, error) { + var ( + apiBindings []*apisv1alpha2.APIBinding + err error + ) + if clusterNameOrWildcard == "*" { + apiBindings, err = indexers.ByIndex[*apisv1alpha2.APIBinding]( + s.Extra.APIBindingInformer.Informer().GetIndexer(), + indexers.APIBindingByIdentityAndGroupResource, + indexers.IdentityGroupResourceKeyFunc(identity, gr.Group, gr.Resource), + ) + } else { + apiBindings, err = indexers.ByIndex[*apisv1alpha2.APIBinding]( + s.Extra.APIBindingInformer.Informer().GetIndexer(), + indexers.APIBindingByBoundResources, + indexers.APIBindingBoundResourceValue(logicalcluster.Name(clusterNameOrWildcard), gr.Group, gr.Resource), + ) + } + if err != nil { + return nil, err + } + + if len(apiBindings) > 0 { + // Matching cluster/identity and bound GR should mean we have the correct APIBinding. + // This is similar to what we're doing in apiBindingAwareCRDLister when selecting + // a binding by identity wildcard. + return apiBindings[0], nil + } + + // This GR does not seem to be provided by an APIBinding. + return nil, nil +} + +func newVirtualResourceHandler(cfg *rest.Config, vwURL, apiExportIdentity, clusterNameOrWildcard string) (http.Handler, error) { + scopedURL, err := url.Parse(virtualResourceURLWithCluster(vwURL, apiExportIdentity, clusterNameOrWildcard)) + if err != nil { + return nil, err + } + + tr, err := rest.TransportFor(cfg) + if err != nil { + return nil, err + } + + proxy := httputil.NewSingleHostReverseProxy(scopedURL) + proxy.Transport = tr + + return proxy, nil +} + +func virtualResourceURLWithCluster(vwURL, apiExportIdentity string, clusterNameOrWildcard string) string { + // Formats the URL like so: + // :/clusters/ + // E.g.: + // /services/replication/1oget0q1249b2vcy/sheriffs:66ac33d6a2baad00af9ba5bac354bf5fed52b29a92d54be33dc94402e8bd9a73/clusters/385doly4poks8a45/apis/wildwest.dev/v1alpha1/sheriffs + return fmt.Sprintf("%s:%s/clusters/%s", vwURL, apiExportIdentity, clusterNameOrWildcard) +} diff --git a/pkg/virtual/framework/virtualresource/context/keys.go b/pkg/virtual/framework/virtualresource/context/keys.go new file mode 100644 index 00000000000..30df5b6ddf6 --- /dev/null +++ b/pkg/virtual/framework/virtualresource/context/keys.go @@ -0,0 +1,38 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package context + +import ( + "context" +) + +type virtualResourceAPIExportIdentityKeyType string + +// virtualResourceAPIExportIdentityKey is a context key that contains the APIExport identity +// of an exported virtual resource. +const virtualResourceAPIExportIdentityKey virtualResourceAPIExportIdentityKeyType = "VirtualResourceAPIExportIdentity" + +// WithVirtualResourceAPIExportIdentity adds the APIExport identity to the context. +func WithVirtualResourceAPIExportIdentity(ctx context.Context, identity string) context.Context { + return context.WithValue(ctx, virtualResourceAPIExportIdentityKey, identity) +} + +// VirtualResourceAPIExportIdentityFrom retrieves the APIExport identity from the context, if any. +func VirtualResourceAPIExportIdentityFrom(ctx context.Context) (string, bool) { + identity, hasIdentity := ctx.Value(virtualResourceAPIExportIdentityKey).(string) + return identity, hasIdentity +} diff --git a/pkg/virtual/replication/authorizer/authorizer.go b/pkg/virtual/replication/authorizer/authorizer.go deleted file mode 100644 index 550c16ef8e4..00000000000 --- a/pkg/virtual/replication/authorizer/authorizer.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package authorizer - -import ( - "context" - "fmt" - "slices" - - "k8s.io/apiserver/pkg/authorization/authorizer" - genericapirequest "k8s.io/apiserver/pkg/endpoints/request" - - kcpkubeclientset "github.com/kcp-dev/client-go/kubernetes" - "github.com/kcp-dev/logicalcluster/v3" - - "github.com/kcp-dev/kcp/pkg/authorization/delegated" - dynamiccontext "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/context" - "github.com/kcp-dev/kcp/pkg/virtual/replication/apidomainkey" -) - -type wrappedResourceAuthorizer struct { - newDelegatedAuthorizer func(clusterName logicalcluster.Name) (authorizer.Authorizer, error) -} - -var readOnlyVerbs = []string{"get", "list", "watch"} - -func NewWrappedResourceAuthorizer(kubeClusterClient kcpkubeclientset.ClusterInterface) authorizer.Authorizer { - return &wrappedResourceAuthorizer{ - newDelegatedAuthorizer: func(clusterName logicalcluster.Name) (authorizer.Authorizer, error) { - return delegated.NewDelegatedAuthorizer(clusterName, kubeClusterClient, delegated.Options{}) - }, - } -} - -func (a *wrappedResourceAuthorizer) Authorize(ctx context.Context, attr authorizer.Attributes) (authorizer.Decision, string, error) { - targetCluster, err := genericapirequest.ValidClusterFrom(ctx) - if err != nil { - return authorizer.DecisionNoOpinion, "", fmt.Errorf("error getting valid cluster from context: %w", err) - } - - parsedKey, err := apidomainkey.Parse(dynamiccontext.APIDomainKeyFrom(ctx)) - if err != nil { - return authorizer.DecisionNoOpinion, "", - fmt.Errorf("invalid API domain key") - } - - if !slices.Contains(readOnlyVerbs, attr.GetVerb()) { - return authorizer.DecisionDeny, "write access to CachedResource is not allowed from virtual workspace", nil - } - - authz, err := a.newDelegatedAuthorizer(targetCluster.Name) - if err != nil { - return authorizer.DecisionNoOpinion, "", err - } - - dec, reason, err := authz.Authorize(ctx, attr) - if err != nil { - return authorizer.DecisionNoOpinion, "", fmt.Errorf("error authorizing RBAC in workspace %q for CachedResource %s|%s: %w", - targetCluster.Name, parsedKey.CachedResourceCluster.String(), parsedKey.CachedResourceName, err) - } - - if dec == authorizer.DecisionAllow { - return authorizer.DecisionAllow, fmt.Sprintf("CachedResource: %s|%s, workspace: %q RBAC decision: %v", - parsedKey.CachedResourceCluster.String(), parsedKey.CachedResourceName, targetCluster.Name, reason), nil - } - - return authorizer.DecisionDeny, fmt.Sprintf("CachedResource: %s|%s, workspace: %q RBAC decision: %v", - parsedKey.CachedResourceCluster.String(), parsedKey.CachedResourceName, targetCluster.Name, reason), nil -} diff --git a/pkg/virtual/replication/authorizer/content.go b/pkg/virtual/replication/authorizer/content.go new file mode 100644 index 00000000000..31dd6ce3994 --- /dev/null +++ b/pkg/virtual/replication/authorizer/content.go @@ -0,0 +1,224 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package authorizer + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authorization/authorizer" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + + kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" + "github.com/kcp-dev/logicalcluster/v3" + + "github.com/kcp-dev/kcp/pkg/authorization/delegated" + "github.com/kcp-dev/kcp/pkg/indexers" + "github.com/kcp-dev/kcp/pkg/informer" + dynamiccontext "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/context" + vrcontext "github.com/kcp-dev/kcp/pkg/virtual/framework/virtualresource/context" + "github.com/kcp-dev/kcp/pkg/virtual/replication/apidomainkey" + apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" + cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" + kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" +) + +type contentAuthorizer struct { + getAPIBindingByIdentityAndGR func(cluster logicalcluster.Name, apiExportIdentity string, gr schema.GroupResource) (*apisv1alpha2.APIBinding, error) + getAPIExportsByVirtualResourceIdentityAndGR func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) + getAPIExportByPath func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) + getCachedResource func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) + + newDelegatedAuthorizer func(cluster logicalcluster.Name) (authorizer.Authorizer, error) +} + +var readOnlyVerbs = sets.New("get", "list", "watch") + +// NewContentAuthorizer creates an authorizer that checks apiexports/content permission +// on relevant APIExports that export the CachedResource in the request URL. +// APIExports must have identity that matches the one specified in the request URL. +func NewContentAuthorizer( + kubeClusterClient kcpkubernetesclientset.ClusterInterface, + localKcpInformers kcpinformers.SharedInformerFactory, + globalKcpInformers kcpinformers.SharedInformerFactory, +) authorizer.Authorizer { + return &contentAuthorizer{ + getAPIBindingByIdentityAndGR: func(cluster logicalcluster.Name, apiExportIdentity string, gr schema.GroupResource) (*apisv1alpha2.APIBinding, error) { + bindings, err := indexers.ByIndex[*apisv1alpha2.APIBinding]( + localKcpInformers.Apis().V1alpha2().APIBindings().Informer().GetIndexer(), + indexers.APIBindingByIdentityAndGroupResource, + indexers.IdentityGroupResourceKeyFunc(apiExportIdentity, gr.Group, gr.Resource), + ) + if err != nil { + return nil, err + } + if len(bindings) == 0 { + return nil, nil + } + return bindings[0], nil + }, + + getAPIExportsByVirtualResourceIdentityAndGR: func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + return indexers.ByIndex[*apisv1alpha2.APIExport]( + globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + indexers.APIExportByVirtualResourceIdentitiesAndGRs, + indexers.VirtualResourceIdentityAndGRKey(vrIdentity, gr), + ) + }, + + getAPIExportByPath: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + return indexers.ByPathAndNameWithFallback[*apisv1alpha2.APIExport]( + apisv1alpha2.Resource("apiexports"), + localKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + path, + name, + ) + }, + + getCachedResource: informer.NewScopedGetterWithFallback( + localKcpInformers.Cache().V1alpha1().CachedResources().Lister(), + globalKcpInformers.Cache().V1alpha1().CachedResources().Lister(), + ), + + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return delegated.NewDelegatedAuthorizer(cluster, kubeClusterClient, delegated.Options{}) + }, + } +} + +func (a *contentAuthorizer) Authorize(ctx context.Context, attr authorizer.Attributes) (authorizer.Decision, string, error) { + if !readOnlyVerbs.Has(attr.GetVerb()) { + return authorizer.DecisionDeny, "write access to Replication virtual workspace is not allowed", nil + } + + parsedKey, err := apidomainkey.Parse(dynamiccontext.APIDomainKeyFrom(ctx)) + if err != nil { + return authorizer.DecisionNoOpinion, "", + fmt.Errorf("invalid API domain key") + } + + targetCluster, err := genericapirequest.ValidClusterFrom(ctx) + if err != nil { + return authorizer.DecisionNoOpinion, "", fmt.Errorf("error getting valid cluster from context: %w", err) + } + + apiExportIdentity, hasAPIExportIdentity := vrcontext.VirtualResourceAPIExportIdentityFrom(ctx) + if !hasAPIExportIdentity { + return authorizer.DecisionNoOpinion, "", fmt.Errorf("APIExport identity missing in context") + } + + cachedResource, err := a.getCachedResource(parsedKey.CachedResourceCluster, parsedKey.CachedResourceName) + if err != nil { + return authorizer.DecisionNoOpinion, "", err + } + + var exports []*apisv1alpha2.APIExport + + if targetCluster.Wildcard || !attr.IsResourceRequest() { + // For non-resource or wildcard requests we need to check all relevant APIExports. + exports, err = a.getAPIExportsByVirtualResourceIdentityAndGR(cachedResource.Status.IdentityHash, schema.GroupResource{ + Group: cachedResource.Spec.Group, + Resource: cachedResource.Spec.Resource, + }) + if err != nil { + return authorizer.DecisionNoOpinion, "", err + } + } else { + // We have a request against a concrete cluster. There should be an associated binding in that cluster. + binding, err := a.getAPIBindingByIdentityAndGR(targetCluster.Name, apiExportIdentity, schema.GroupResource{ + Group: cachedResource.Spec.Group, + Resource: cachedResource.Spec.Resource, + }) + if err != nil || binding == nil { + return authorizer.DecisionDeny, "could not find suitable APIBinding in target logical cluster", nil //nolint:nilerr // this is on purpose, we want to deny, not return a server error + } + path := logicalcluster.NewPath(binding.Spec.Reference.Export.Path) + if path.Empty() { + path = logicalcluster.From(binding).Path() + } + export, err := a.getAPIExportByPath(path, binding.Spec.Reference.Export.Name) + if err != nil { + return authorizer.DecisionNoOpinion, "APIExport not found", err + } + exports = append(exports, export) + } + + // Make sure the user has apiexports/content permissions to the exports that refer to this CachedResource resource. + + SARAttributes := authorizer.AttributesRecord{ + APIGroup: apisv1alpha1.SchemeGroupVersion.Group, + APIVersion: apisv1alpha1.SchemeGroupVersion.Version, + User: attr.GetUser(), + Verb: attr.GetVerb(), + Resource: "apiexports", + ResourceRequest: true, + Subresource: "content", + } + + var seenCachedResourceReference bool + for _, export := range exports { + if export.Status.IdentityHash != apiExportIdentity { + // getAPIExportsByVirtualResourceIdentityAndGR has returned APIExports + // that don't include the APIExport identity we've received in the request URL. + continue + } + + for _, resource := range export.Spec.Resources { + if resource.Storage.Virtual == nil || + resource.Storage.Virtual.IdentityHash != cachedResource.Status.IdentityHash { + continue + } + if resource.Group != cachedResource.Spec.Group || + resource.Name != cachedResource.Spec.Resource { + continue + } + if resource.Storage.Virtual.Reference.APIGroup == nil || + *resource.Storage.Virtual.Reference.APIGroup != cachev1alpha1.SchemeGroupVersion.Group || + resource.Storage.Virtual.Reference.Kind != "CachedResourceEndpointSlice" || + resource.Storage.Virtual.Reference.Name != cachedResource.Name { + continue + } + + authz, err := a.newDelegatedAuthorizer(logicalcluster.From(export)) + if err != nil { + return authorizer.DecisionNoOpinion, "", + fmt.Errorf("error creating delegated authorizer for API export %q, workspace %q: %w", export.Name, logicalcluster.From(export), err) + } + SARAttributes.Name = export.Name + dec, reason, err := authz.Authorize(ctx, SARAttributes) + if err != nil { + return authorizer.DecisionNoOpinion, "", + fmt.Errorf("error authorizing RBAC in API export %q, workspace %q: %w", export.Name, logicalcluster.From(export), err) + } + if dec != authorizer.DecisionAllow { + return authorizer.DecisionDeny, reason, nil + } + + seenCachedResourceReference = true + } + } + + if !seenCachedResourceReference { + return authorizer.DecisionDeny, "failed to find suitable reason to allow access to CachedResource", nil + } + + return authorizer.DecisionAllow, "found CachedResource reference", nil +} diff --git a/pkg/virtual/replication/authorizer/content_test.go b/pkg/virtual/replication/authorizer/content_test.go new file mode 100644 index 00000000000..b5f35f742df --- /dev/null +++ b/pkg/virtual/replication/authorizer/content_test.go @@ -0,0 +1,668 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package authorizer + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/authorization/authorizer" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + + "github.com/kcp-dev/logicalcluster/v3" + + dynamiccontext "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/context" + vrcontext "github.com/kcp-dev/kcp/pkg/virtual/framework/virtualresource/context" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" + cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" +) + +type alwaysDenyAuthrizer struct{} + +func (*alwaysDenyAuthrizer) Authorize(ctx context.Context, attr authorizer.Attributes) (authorizer.Decision, string, error) { + return authorizer.DecisionDeny, "alwaysDeny", nil +} + +type alwaysAllowAuthrizer struct{} + +func (*alwaysAllowAuthrizer) Authorize(ctx context.Context, attr authorizer.Attributes) (authorizer.Decision, string, error) { + return authorizer.DecisionAllow, "alwaysAllow", nil +} + +func TestContentAuthorizer(t *testing.T) { + tests := map[string]struct { + a contentAuthorizer + ctx context.Context //nolint:containedctx // Mock ctx needed by Authorizer(). + attr authorizer.Attributes + + expectedDecision authorizer.Decision + expectedReason string + expectedErrorStr string + }{ + "non-readonly verbs should fail": { + attr: authorizer.AttributesRecord{ + Verb: "create", + }, + expectedDecision: authorizer.DecisionDeny, + expectedReason: "write access to Replication virtual workspace is not allowed", + }, + "missing API domain key in context": { + ctx: context.Background(), + attr: authorizer.AttributesRecord{ + Verb: "get", + }, + expectedDecision: authorizer.DecisionNoOpinion, + expectedErrorStr: "invalid API domain key", + }, + "missing target cluster in context": { + ctx: dynamiccontext.WithAPIDomainKey(context.Background(), "CachedResourceCluster/cachedresource-1"), + attr: authorizer.AttributesRecord{ + Verb: "get", + }, + expectedDecision: authorizer.DecisionNoOpinion, + expectedErrorStr: "error getting valid cluster from context: no cluster in the request context", + }, + "missing APIExport identity in context": { + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + context.Background(), genericapirequest.Cluster{Name: "TargetCluster"}, + ), + "CachedResourceCluster/cachedresource-1", + ), + attr: authorizer.AttributesRecord{ + Verb: "get", + }, + expectedDecision: authorizer.DecisionNoOpinion, + expectedErrorStr: "APIExport identity missing in context", + }, + "missing CachedResource": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return nil, apierrors.NewNotFound(cachev1alpha1.Resource("cachedresources"), name) + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Name: "TargetCluster"}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionNoOpinion, + expectedErrorStr: `cachedresources.cache.kcp.io "cachedresource-1" not found`, + }, + "wildcard request and no matches": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{}, nil + }, + getAPIExportsByVirtualResourceIdentityAndGR: func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + return nil, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Wildcard: true}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "failed to find suitable reason to allow access to CachedResource", + }, + "wildcard request and APIExport with different identity": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{}, nil + }, + getAPIExportsByVirtualResourceIdentityAndGR: func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + return []*apisv1alpha2.APIExport{ + { + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "SomeOtherAPIExportIdentity", + }, + }, + }, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Wildcard: true}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "failed to find suitable reason to allow access to CachedResource", + }, + "wildcard request and APIExport with different CachedResource": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIExportsByVirtualResourceIdentityAndGR: func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + return []*apisv1alpha2.APIExport{ + { + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "group", + Name: "resource", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: &cachev1alpha1.SchemeGroupVersion.Group, + Kind: "CachedResourceEndpointSlice", + Name: "SomeOtherCachedResource", + }, + IdentityHash: "CachedResourceIdentity-1", + }, + }, + }, + }, + }, + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "APIExportIdentity", + }, + }, + }, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Wildcard: true}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "failed to find suitable reason to allow access to CachedResource", + }, + "wildcard request and deny": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "group", + Resource: "resource", + }, + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIExportsByVirtualResourceIdentityAndGR: func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + return []*apisv1alpha2.APIExport{ + { + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "group", + Name: "resource", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: &cachev1alpha1.SchemeGroupVersion.Group, + Kind: "CachedResourceEndpointSlice", + Name: "cachedresource-1", + }, + IdentityHash: "CachedResourceIdentity-1", + }, + }, + }, + }, + }, + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "APIExportIdentity", + }, + }, + }, nil + }, + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return &alwaysDenyAuthrizer{}, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Wildcard: true}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "alwaysDeny", + }, + "wildcard request and allow": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "group", + Resource: "resource", + }, + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIExportsByVirtualResourceIdentityAndGR: func(vrIdentity string, gr schema.GroupResource) ([]*apisv1alpha2.APIExport, error) { + return []*apisv1alpha2.APIExport{ + { + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "group", + Name: "resource", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: &cachev1alpha1.SchemeGroupVersion.Group, + Kind: "CachedResourceEndpointSlice", + Name: "cachedresource-1", + }, + IdentityHash: "CachedResourceIdentity-1", + }, + }, + }, + }, + }, + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "APIExportIdentity", + }, + }, + }, nil + }, + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return &alwaysAllowAuthrizer{}, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Wildcard: true}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionAllow, + expectedReason: "found CachedResource reference", + }, + "cluster request and no APIBinding": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "group", + Resource: "resource", + }, + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIBindingByIdentityAndGR: func(cluster logicalcluster.Name, apiExportIdentity string, gr schema.GroupResource) (*apisv1alpha2.APIBinding, error) { + return nil, nil + }, + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return &alwaysAllowAuthrizer{}, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + APIGroup: "group", + APIVersion: "v1", + Resource: "resources", + ResourceRequest: true, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Name: "TargetCluster"}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "could not find suitable APIBinding in target logical cluster", + }, + "cluster request and APIExport with different CachedResource": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "group", + Resource: "resource", + }, + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIBindingByIdentityAndGR: func(cluster logicalcluster.Name, apiExportIdentity string, gr schema.GroupResource) (*apisv1alpha2.APIBinding, error) { + return &apisv1alpha2.APIBinding{ + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: "root:provider", + Name: "apiexport-1", + }, + }, + }, + }, nil + }, + getAPIExportByPath: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + m := map[logicalcluster.Path]*apisv1alpha2.APIExport{ + logicalcluster.NewPath("root:provider"): { + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "group", + Name: "resource", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: &cachev1alpha1.SchemeGroupVersion.Group, + Kind: "CachedResourceEndpointSlice", + Name: "SomeOtherCachedResource", + }, + IdentityHash: "CachedResourceIdentity-1", + }, + }, + }, + }, + }, + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "APIExportIdentity", + }, + }, + } + export, found := m[path] + if !found { + return nil, apierrors.NewNotFound(apisv1alpha2.Resource("apiexports"), name) + } + return export, nil + }, + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return &alwaysDenyAuthrizer{}, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + APIGroup: "group", + APIVersion: "v1", + Resource: "resources", + ResourceRequest: true, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Name: "TargetCluster"}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "failed to find suitable reason to allow access to CachedResource", + }, + "cluster request and deny": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "group", + Resource: "resource", + }, + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIBindingByIdentityAndGR: func(cluster logicalcluster.Name, apiExportIdentity string, gr schema.GroupResource) (*apisv1alpha2.APIBinding, error) { + return &apisv1alpha2.APIBinding{ + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: "root:provider", + Name: "apiexport-1", + }, + }, + }, + }, nil + }, + getAPIExportByPath: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + m := map[logicalcluster.Path]*apisv1alpha2.APIExport{ + logicalcluster.NewPath("root:provider"): { + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "group", + Name: "resource", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: &cachev1alpha1.SchemeGroupVersion.Group, + Kind: "CachedResourceEndpointSlice", + Name: "cachedresource-1", + }, + IdentityHash: "CachedResourceIdentity-1", + }, + }, + }, + }, + }, + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "APIExportIdentity", + }, + }, + } + export, found := m[path] + if !found { + return nil, apierrors.NewNotFound(apisv1alpha2.Resource("apiexports"), name) + } + return export, nil + }, + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return &alwaysDenyAuthrizer{}, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + APIGroup: "group", + APIVersion: "v1", + Resource: "resources", + ResourceRequest: true, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Name: "TargetCluster"}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionDeny, + expectedReason: "alwaysDeny", + }, + "cluster request and allow": { + a: contentAuthorizer{ + getCachedResource: func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) { + return &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cachedresource-1", + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "group", + Resource: "resource", + }, + }, + Status: cachev1alpha1.CachedResourceStatus{ + IdentityHash: "CachedResourceIdentity-1", + }, + }, nil + }, + getAPIBindingByIdentityAndGR: func(cluster logicalcluster.Name, apiExportIdentity string, gr schema.GroupResource) (*apisv1alpha2.APIBinding, error) { + return &apisv1alpha2.APIBinding{ + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: "root:provider", + Name: "apiexport-1", + }, + }, + }, + }, nil + }, + getAPIExportByPath: func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) { + m := map[logicalcluster.Path]*apisv1alpha2.APIExport{ + logicalcluster.NewPath("root:provider"): { + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "group", + Name: "resource", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: &cachev1alpha1.SchemeGroupVersion.Group, + Kind: "CachedResourceEndpointSlice", + Name: "cachedresource-1", + }, + IdentityHash: "CachedResourceIdentity-1", + }, + }, + }, + }, + }, + Status: apisv1alpha2.APIExportStatus{ + IdentityHash: "APIExportIdentity", + }, + }, + } + export, found := m[path] + if !found { + return nil, apierrors.NewNotFound(apisv1alpha2.Resource("apiexports"), name) + } + return export, nil + }, + newDelegatedAuthorizer: func(cluster logicalcluster.Name) (authorizer.Authorizer, error) { + return &alwaysAllowAuthrizer{}, nil + }, + }, + attr: authorizer.AttributesRecord{ + Verb: "get", + User: &user.DefaultInfo{}, + APIGroup: "group", + APIVersion: "v1", + Resource: "resources", + ResourceRequest: true, + }, + ctx: dynamiccontext.WithAPIDomainKey( + genericapirequest.WithCluster( + vrcontext.WithVirtualResourceAPIExportIdentity( + context.Background(), "APIExportIdentity", + ), genericapirequest.Cluster{Name: "TargetCluster"}, + ), + "CachedResourceCluster/cachedresource-1", + ), + expectedDecision: authorizer.DecisionAllow, + expectedReason: "found CachedResource reference", + }, + } + for tname, tt := range tests { + t.Run(tname, func(t *testing.T) { + dec, reason, err := tt.a.Authorize(tt.ctx, tt.attr) + if tt.expectedErrorStr == "" { + require.NoError(t, err, "was not expecting to return an error") + } else { + require.Equal(t, tt.expectedErrorStr, err.Error(), "unexpected error") + } + require.Equal(t, tt.expectedDecision, dec, "unexpected decision") + require.Equal(t, tt.expectedReason, reason, "unexpected reason") + }) + } +} diff --git a/pkg/virtual/replication/builder/build.go b/pkg/virtual/replication/builder/build.go index 5fafd5f398d..b204e2087a3 100644 --- a/pkg/virtual/replication/builder/build.go +++ b/pkg/virtual/replication/builder/build.go @@ -23,6 +23,8 @@ import ( "strings" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authorization/authorizer" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" genericapiserver "k8s.io/apiserver/pkg/server" @@ -37,7 +39,6 @@ import ( "github.com/kcp-dev/kcp/pkg/authorization" "github.com/kcp-dev/kcp/pkg/indexers" "github.com/kcp-dev/kcp/pkg/informer" - cachedresources "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresources" cachedresourcesreplication "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresources/replication" "github.com/kcp-dev/kcp/pkg/virtual/framework" virtualworkspacesdynamic "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic" @@ -46,19 +47,47 @@ import ( dynamiccontext "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/context" "github.com/kcp-dev/kcp/pkg/virtual/framework/forwardingregistry" "github.com/kcp-dev/kcp/pkg/virtual/framework/rootapiserver" + vrcontext "github.com/kcp-dev/kcp/pkg/virtual/framework/virtualresource/context" "github.com/kcp-dev/kcp/pkg/virtual/replication" "github.com/kcp-dev/kcp/pkg/virtual/replication/apidomainkey" replicationauthorizer "github.com/kcp-dev/kcp/pkg/virtual/replication/authorizer" apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/core" corev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/core/v1alpha1" "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" + apisv1alpha2informers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions/apis/v1alpha2" apisv1alpha1listers "github.com/kcp-dev/kcp/sdk/client/listers/apis/v1alpha1" cachev1alpha1listers "github.com/kcp-dev/kcp/sdk/client/listers/cache/v1alpha1" ) +func listAPIBindingsByCachedResource(identityHash string, gr schema.GroupResource, globalAPIExportIndexer cache.Indexer, apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer) ([]*apisv1alpha2.APIBinding, error) { + exports, err := indexers.ByIndex[*apisv1alpha2.APIExport](globalAPIExportIndexer, indexers.APIExportByVirtualResourceIdentitiesAndGRs, indexers.VirtualResourceIdentityAndGRKey(identityHash, gr)) + if err != nil { + return nil, err + } + + var bindings []*apisv1alpha2.APIBinding + for _, export := range exports { + exportBindings, err := listAPIBindingsByAPIExport(apiBindingInformer, export) + if err != nil { + return nil, err + } + bindings = append(bindings, exportBindings...) + } + return bindings, err +} + +func listClustersInBindings(bindings []*apisv1alpha2.APIBinding) sets.Set[logicalcluster.Name] { + s := sets.New[logicalcluster.Name]() + for _, binding := range bindings { + s.Insert(logicalcluster.From(binding)) + } + return s +} + func BuildVirtualWorkspace( cfg *rest.Config, rootPathPrefix string, @@ -75,31 +104,21 @@ func BuildVirtualWorkspace( cachedResourceContent := &virtualworkspacesdynamic.DynamicVirtualWorkspace{ RootPathResolver: framework.RootPathResolverFunc(func(urlPath string, requestContext context.Context) (accepted bool, prefixToStrip string, completedContext context.Context) { - targetCluster, apiDomain, prefixToStrip, ok := digestURL(urlPath, rootPathPrefix) + targetCluster, apiDomain, prefixToStrip, apiExportIdentity, ok := digestURL(urlPath, rootPathPrefix) if !ok { return false, "", requestContext } - - if targetCluster.Wildcard { - return false, "", requestContext - } - - parsedKey, err := apidomainkey.Parse(apiDomain) + _, err := apidomainkey.Parse(apiDomain) if err != nil { return false, "", requestContext } - if targetCluster.Name != parsedKey.CachedResourceCluster { - return false, "", requestContext - } - - // We only accept requests for CachedResource's local cluster. - completedContext = genericapirequest.WithCluster(requestContext, targetCluster) completedContext = dynamiccontext.WithAPIDomainKey(completedContext, apiDomain) + completedContext = vrcontext.WithVirtualResourceAPIExportIdentity(completedContext, apiExportIdentity) return true, prefixToStrip, completedContext }), - Authorizer: newAuth(kubeClusterClient), + Authorizer: newAuthorizer(kubeClusterClient, localKcpInformers, globalKcpInformers), ReadyChecker: framework.ReadyFunc(func() error { select { case <-readyCh: @@ -122,37 +141,48 @@ func BuildVirtualWorkspace( localInformers := map[string]cache.SharedIndexInformer{ "cachedresources": localKcpInformers.Cache().V1alpha1().CachedResources().Informer(), "apiexports": localKcpInformers.Apis().V1alpha2().APIExports().Informer(), + "apibindings": localKcpInformers.Apis().V1alpha2().APIBindings().Informer(), "apiresourceschemas": localKcpInformers.Apis().V1alpha1().APIResourceSchemas().Informer(), } - if err := mainConfig.AddPostStartHook(replication.VirtualWorkspaceName, func(hookContext genericapiserver.PostStartHookContext) error { - defer close(readyCh) - - // CachedResources indexers. + // Install indexers. - indexers.AddIfNotPresentOrDie( - globalKcpInformers.Cache().V1alpha1().CachedObjects().Informer().GetIndexer(), - cache.Indexers{ - cachedresourcesreplication.ByGVRAndLogicalClusterAndNamespace: cachedresourcesreplication.IndexByGVRAndLogicalClusterAndNamespace, - }, - ) + // CachedResources indexers. + indexers.AddIfNotPresentOrDie( + globalKcpInformers.Cache().V1alpha1().CachedObjects().Informer().GetIndexer(), + cache.Indexers{ + cachedresourcesreplication.ByGVRAndLogicalClusterAndNamespace: cachedresourcesreplication.IndexByGVRAndLogicalClusterAndNamespace, + }, + ) + + // APIExport indexers. + indexers.AddIfNotPresentOrDie( + globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + cache.Indexers{ + indexers.APIExportByIdentity: indexers.IndexAPIExportByIdentity, + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + indexers.APIExportByVirtualResourceIdentitiesAndGRs: indexers.IndexAPIExportByVirtualResourceIdentitiesAndGRs, + }, + ) + indexers.AddIfNotPresentOrDie( + localKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), + cache.Indexers{ + indexers.APIExportByIdentity: indexers.IndexAPIExportByIdentity, + indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, + indexers.APIExportByVirtualResourceIdentitiesAndGRs: indexers.IndexAPIExportByVirtualResourceIdentitiesAndGRs, + }, + ) - // APIExport indexers. + // APIBinding indexers. + indexers.AddIfNotPresentOrDie(localKcpInformers.Apis().V1alpha2().APIBindings().Informer().GetIndexer(), cache.Indexers{ + indexers.APIBindingsByAPIExport: indexers.IndexAPIBindingByAPIExport, + indexers.APIBindingByIdentityAndGroupResource: indexers.IndexAPIBindingByIdentityGroupResource, + }) - indexers.AddIfNotPresentOrDie( - globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), - cache.Indexers{ - indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, - }, - ) - indexers.AddIfNotPresentOrDie( - localKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), - cache.Indexers{ - indexers.ByLogicalClusterPathAndName: indexers.IndexByLogicalClusterPathAndName, - }, - ) + // Wait for caches to be synced. - // Wait for caches to be synced. + if err := mainConfig.AddPostStartHook(replication.VirtualWorkspaceName, func(hookContext genericapiserver.PostStartHookContext) error { + defer close(readyCh) for name, informer := range globalInformers { if !cache.WaitForNamedCacheSync(name, hookContext.Done(), informer.HasSynced) { @@ -199,13 +229,17 @@ func BuildVirtualWorkspace( getCachedResource: informer.NewScopedGetterWithFallback[*cachev1alpha1.CachedResource, cachev1alpha1listers.CachedResourceLister](localKcpInformers.Cache().V1alpha1().CachedResources().Lister(), globalKcpInformers.Cache().V1alpha1().CachedResources().Lister()), + getAPIExportsByIdentity: func(identity string) ([]*apisv1alpha2.APIExport, error) { + return indexers.ByIndex[*apisv1alpha2.APIExport](globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), indexers.APIExportByIdentity, identity) + }, + config: mainConfig, dynamicClusterClient: dynamicClusterClient, - storageProvider: func(ctx context.Context, dynamicClusterClientFunc forwardingregistry.DynamicClusterClientFunc, apiResourceSchema *apisv1alpha1.APIResourceSchema, version string) (apiserver.RestProviderFunc, error) { + storageProvider: func(ctx context.Context, dynamicClusterClientFunc forwardingregistry.DynamicClusterClientFunc, apiResourceSchema *apisv1alpha1.APIResourceSchema, version string, cr *cachev1alpha1.CachedResource) (apiserver.RestProviderFunc, error) { return forwardingregistry.ProvideReadOnlyRestStorage( ctx, dynamicClusterClientFunc, - withUnwrapping(apiResourceSchema, version, globalKcpInformers), + withUnwrapping(apiResourceSchema, version, localKcpInformers, globalKcpInformers, cr), nil, ) }, @@ -218,14 +252,47 @@ func BuildVirtualWorkspace( }, nil } +func listAPIBindingsByAPIExport(apiBindingInformer apisv1alpha2informers.APIBindingClusterInformer, export *apisv1alpha2.APIExport) ([]*apisv1alpha2.APIBinding, error) { + // binding keys by full path + keys := sets.New[string]() + if path := logicalcluster.NewPath(export.Annotations[core.LogicalClusterPathAnnotationKey]); !path.Empty() { + pathKeys, err := apiBindingInformer.Informer().GetIndexer().IndexKeys(indexers.APIBindingsByAPIExport, path.Join(export.Name).String()) + if err != nil { + return nil, err + } + keys.Insert(pathKeys...) + } + + clusterKeys, err := apiBindingInformer.Informer().GetIndexer().IndexKeys(indexers.APIBindingsByAPIExport, logicalcluster.From(export).Path().Join(export.Name).String()) + if err != nil { + return nil, err + } + keys.Insert(clusterKeys...) + + bindings := make([]*apisv1alpha2.APIBinding, 0, keys.Len()) + for _, key := range sets.List[string](keys) { + binding, exists, err := apiBindingInformer.Informer().GetIndexer().GetByKey(key) + if err != nil { + utilruntime.HandleError(err) + continue + } else if !exists { + utilruntime.HandleError(fmt.Errorf("APIBinding %q does not exist", key)) + continue + } + bindings = append(bindings, binding.(*apisv1alpha2.APIBinding)) + } + return bindings, nil +} + func digestURL(urlPath, rootPathPrefix string) ( cluster genericapirequest.Cluster, key dynamiccontext.APIDomainKey, logicalPath string, + apiExportIdentity string, accepted bool, ) { if !strings.HasPrefix(urlPath, rootPathPrefix) { - return genericapirequest.Cluster{}, "", "", false + return genericapirequest.Cluster{}, "", "", "", false } // Incoming requests to this virtual workspace will look like: @@ -236,17 +303,23 @@ func digestURL(urlPath, rootPathPrefix string) ( parts := strings.SplitN(withoutRootPathPrefix, "/", 3) if len(parts) < 3 { - return genericapirequest.Cluster{}, "", "", false + return genericapirequest.Cluster{}, "", "", "", false } - cachedResourceClusterName, cachedResourceName := parts[0], parts[1] + cachedResourceClusterName, cachedResourceNameAndIdentity := parts[0], parts[1] if cachedResourceClusterName == "" { - return genericapirequest.Cluster{}, "", "", false + return genericapirequest.Cluster{}, "", "", "", false } - if cachedResourceName == "" { - return genericapirequest.Cluster{}, "", "", false + if cachedResourceNameAndIdentity == "" { + return genericapirequest.Cluster{}, "", "", "", false } + cachedResourceNameAndIdentityParts := strings.Split(cachedResourceNameAndIdentity, ":") + if len(cachedResourceNameAndIdentityParts) != 2 { + return genericapirequest.Cluster{}, "", "", "", false + } + cachedResourceName, apiExportIdentity := cachedResourceNameAndIdentityParts[0], cachedResourceNameAndIdentityParts[1] + realPath := "/" if len(parts) > 2 { realPath += parts[2] @@ -257,7 +330,7 @@ func digestURL(urlPath, rootPathPrefix string) ( // We are now here: ───┘ // Now, we parse out the logical cluster. if !strings.HasPrefix(realPath, "/clusters/") { - return genericapirequest.Cluster{}, "", "", false + return genericapirequest.Cluster{}, "", "", "", false } withoutClustersPrefix := strings.TrimPrefix(realPath, "/clusters/") @@ -275,19 +348,23 @@ func digestURL(urlPath, rootPathPrefix string) ( var ok bool cluster.Name, ok = path.Name() if !ok { - return genericapirequest.Cluster{}, "", "", false + return genericapirequest.Cluster{}, "", "", "", false } } key = apidomainkey.New(logicalcluster.Name(cachedResourceClusterName), cachedResourceName) - return cluster, key, strings.TrimSuffix(urlPath, realPath), true + return cluster, key, strings.TrimSuffix(urlPath, realPath), apiExportIdentity, true } -func newAuth(deepSARClient kcpkubernetesclientset.ClusterInterface) authorizer.Authorizer { - wrappedResourceAuthorizer := replicationauthorizer.NewWrappedResourceAuthorizer(deepSARClient) - wrappedResourceAuthorizer = authorization.NewDecorator("virtual.replication.wrappedresource.authorization.kcp.io", wrappedResourceAuthorizer).AddAuditLogging().AddAnonymization().AddReasonAnnotation() +func newAuthorizer( + kubeClusterClient kcpkubernetesclientset.ClusterInterface, + localKcpInformers kcpinformers.SharedInformerFactory, + globalKcpInformers kcpinformers.SharedInformerFactory, +) authorizer.Authorizer { + contentAuthorizer := replicationauthorizer.NewContentAuthorizer(kubeClusterClient, localKcpInformers, globalKcpInformers) + contentAuthorizer = authorization.NewDecorator("virtual.replication.content.authorization.kcp.io", contentAuthorizer).AddAuditLogging().AddAnonymization().AddReasonAnnotation() - return wrappedResourceAuthorizer + return contentAuthorizer } var _ apidefinition.APIDefinitionSetGetter = &singleResourceAPIDefinitionSetProvider{} @@ -295,44 +372,100 @@ var _ apidefinition.APIDefinitionSetGetter = &singleResourceAPIDefinitionSetProv type singleResourceAPIDefinitionSetProvider struct { config genericapiserver.CompletedConfig dynamicClusterClient kcpdynamic.ClusterInterface - storageProvider func(ctx context.Context, dynamicClusterClientFunc forwardingregistry.DynamicClusterClientFunc, sch *apisv1alpha1.APIResourceSchema, version string) (apiserver.RestProviderFunc, error) + storageProvider func(ctx context.Context, dynamicClusterClientFunc forwardingregistry.DynamicClusterClientFunc, apiResourceSchema *apisv1alpha1.APIResourceSchema, version string, cr *cachev1alpha1.CachedResource) (apiserver.RestProviderFunc, error) localKcpInformers kcpinformers.SharedInformerFactory globalKcpInformers kcpinformers.SharedInformerFactory - getLogicalCluster func(cluster logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) - getAPIBinding func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) - getAPIExportByPath func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) - getCachedResource func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) - getAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) + getLogicalCluster func(cluster logicalcluster.Name, name string) (*corev1alpha1.LogicalCluster, error) + getAPIBinding func(cluster logicalcluster.Name, name string) (*apisv1alpha2.APIBinding, error) + getAPIExportByPath func(path logicalcluster.Path, name string) (*apisv1alpha2.APIExport, error) + getCachedResource func(cluster logicalcluster.Name, name string) (*cachev1alpha1.CachedResource, error) + getAPIExportsByIdentity func(identity string) ([]*apisv1alpha2.APIExport, error) + getAPIResourceSchema func(cluster logicalcluster.Name, name string) (*apisv1alpha1.APIResourceSchema, error) } func (a *singleResourceAPIDefinitionSetProvider) GetAPIDefinitionSet(ctx context.Context, key dynamiccontext.APIDomainKey) (apis apidefinition.APIDefinitionSet, apisExist bool, err error) { + // TODO: consider making this into a controller. + parsedKey, err := apidomainkey.Parse(key) if err != nil { return nil, false, err } - clientFactory := func(ctx context.Context) (kcpdynamic.ClusterInterface, error) { - return a.dynamicClusterClient, nil + exportIdentity, hasExportIdentity := vrcontext.VirtualResourceAPIExportIdentityFrom(ctx) + if !hasExportIdentity { + return nil, false, nil } cachedResource, err := a.getCachedResource(parsedKey.CachedResourceCluster, parsedKey.CachedResourceName) if err != nil { return nil, false, err } - if !conditions.IsTrue(cachedResource, cachev1alpha1.CachedResourceValid) { return nil, false, fmt.Errorf("CachedResource %s|%s not ready", parsedKey.CachedResourceCluster, parsedKey.CachedResourceName) } - wrappedGVR := schema.GroupVersionResource(cachedResource.Spec.GroupVersionResource) - wrappedSch, err := a.getAPIResourceSchema(logicalcluster.From(cachedResource), cachedresources.CachedAPIResourceSchemaName(cachedResource.UID, wrappedGVR.GroupResource())) + candidateExports, err := a.getAPIExportsByIdentity(exportIdentity) if err != nil { - return nil, false, fmt.Errorf("failed to get schema for wrapped object in CachedResource %s|%s: %v", parsedKey.CachedResourceCluster, parsedKey.CachedResourceName, err) + return nil, false, err + } + + wrappedGVR := schema.GroupVersionResource(cachedResource.Spec.GroupVersionResource) + var wrappedSch *apisv1alpha1.APIResourceSchema + + for _, export := range candidateExports { + for _, res := range export.Spec.Resources { + if res.Group != wrappedGVR.Group || res.Name != wrappedGVR.Resource { + continue + } + if res.Storage.Virtual == nil { + continue + } + if res.Storage.Virtual.IdentityHash != cachedResource.Status.IdentityHash { + continue + } + + sch, err := a.getAPIResourceSchema(logicalcluster.From(export), res.Schema) + if err != nil { + return nil, false, fmt.Errorf("failed to get APIResourceSchema: %v", err) + } + + if sch.Spec.Group != wrappedGVR.Group { + continue + } + if sch.Spec.Names.Plural != wrappedGVR.Resource { + continue + } + + hasVersion := false + for _, schVersion := range sch.Spec.Versions { + if schVersion.Name != wrappedGVR.Version { + continue + } + if !schVersion.Served { + continue + } + + hasVersion = true + break + } + + if hasVersion { + wrappedSch = sch + break + } + } + } + if wrappedSch == nil { + return nil, false, fmt.Errorf("failed to get schema for wrapped object in CachedResource %s|%s: missing schema", parsedKey.CachedResourceCluster, parsedKey.CachedResourceName) + } + + clientFactory := func(ctx context.Context) (kcpdynamic.ClusterInterface, error) { + return a.dynamicClusterClient, nil } - restProvider, err := a.storageProvider(ctx, clientFactory, wrappedSch, wrappedGVR.Version) + restProvider, err := a.storageProvider(ctx, clientFactory, wrappedSch, wrappedGVR.Version, cachedResource) if err != nil { return nil, false, err } diff --git a/pkg/virtual/replication/builder/build_test.go b/pkg/virtual/replication/builder/build_test.go index abfb41fc3c3..db4d1ca832a 100644 --- a/pkg/virtual/replication/builder/build_test.go +++ b/pkg/virtual/replication/builder/build_test.go @@ -29,32 +29,44 @@ import ( func TestDigestUrl(t *testing.T) { rootPathPrefix := "/services/replication/" testCases := []struct { - urlPath string - expectedAccept bool - expectedCluster genericapirequest.Cluster - expectedKey context.APIDomainKey - expectedLogicalPath string + urlPath string + expectedAccept bool + expectedCluster genericapirequest.Cluster + expectedKey context.APIDomainKey + expectedAPIExportIdentity string + expectedLogicalPath string }{ { - urlPath: "/services/replication/my-cluster/my-cachedresource/clusters/my-cluster/apis", - expectedAccept: true, - expectedKey: "my-cluster/my-cachedresource", - expectedCluster: genericapirequest.Cluster{Name: "my-cluster", Wildcard: false}, - expectedLogicalPath: "/services/replication/my-cluster/my-cachedresource/clusters/my-cluster", + urlPath: "/services/replication/my-cluster/my-cachedresource:123abc/clusters/my-cluster/apis", + expectedAccept: true, + expectedKey: "my-cluster/my-cachedresource", + expectedCluster: genericapirequest.Cluster{Name: "my-cluster", Wildcard: false}, + expectedAPIExportIdentity: "123abc", + expectedLogicalPath: "/services/replication/my-cluster/my-cachedresource:123abc/clusters/my-cluster", }, { - urlPath: "/services/replication/my-cluster/my-cachedresource/clusters/my-cluster", - expectedAccept: true, - expectedKey: "my-cluster/my-cachedresource", - expectedCluster: genericapirequest.Cluster{Name: "my-cluster", Wildcard: false}, - expectedLogicalPath: "/services/replication/my-cluster/my-cachedresource/clusters/my-cluster", + urlPath: "/services/replication/my-cluster/my-cachedresource:123abc/clusters/my-cluster", + expectedAccept: true, + expectedKey: "my-cluster/my-cachedresource", + expectedCluster: genericapirequest.Cluster{Name: "my-cluster", Wildcard: false}, + expectedAPIExportIdentity: "123abc", + expectedLogicalPath: "/services/replication/my-cluster/my-cachedresource:123abc/clusters/my-cluster", }, { - urlPath: "/services/replication/my-cluster/my-cachedresource/clusters/other-cluster", - expectedAccept: true, - expectedKey: "my-cluster/my-cachedresource", - expectedCluster: genericapirequest.Cluster{Name: "other-cluster", Wildcard: false}, - expectedLogicalPath: "/services/replication/my-cluster/my-cachedresource/clusters/other-cluster", + urlPath: "/services/replication/my-cluster/my-cachedresource:123abc/clusters/other-cluster", + expectedAccept: true, + expectedKey: "my-cluster/my-cachedresource", + expectedCluster: genericapirequest.Cluster{Name: "other-cluster", Wildcard: false}, + expectedAPIExportIdentity: "123abc", + expectedLogicalPath: "/services/replication/my-cluster/my-cachedresource:123abc/clusters/other-cluster", + }, + { + urlPath: "/services/replication/my-cluster/my-cachedresource/clusters/my-cluster/apis", + expectedAccept: false, + expectedKey: "", + expectedCluster: genericapirequest.Cluster{}, + expectedAPIExportIdentity: "", + expectedLogicalPath: "", }, { urlPath: "/services/replication/my-cluster/my-cachedresource/clusters", @@ -67,10 +79,11 @@ func TestDigestUrl(t *testing.T) { for _, tc := range testCases { t.Run(tc.urlPath, func(t *testing.T) { - clusterName, key, logicalPath, accepted := digestURL(tc.urlPath, rootPathPrefix) + clusterName, key, logicalPath, apiExportIdentity, accepted := digestURL(tc.urlPath, rootPathPrefix) require.Equal(t, tc.expectedAccept, accepted, "Accepted should match expected value") require.Equal(t, tc.expectedKey, key, "Key should match expected value") require.Equal(t, tc.expectedCluster, clusterName, "cluster name should match expected value") + require.Equal(t, tc.expectedAPIExportIdentity, apiExportIdentity, "APIExport identity should match expected value") require.Equal(t, tc.expectedLogicalPath, logicalPath, "LogicalPath should match expected value") }) } diff --git a/pkg/virtual/replication/builder/unwrap.go b/pkg/virtual/replication/builder/unwrap.go index 54b9efe27f9..27a4e0fc378 100644 --- a/pkg/virtual/replication/builder/unwrap.go +++ b/pkg/virtual/replication/builder/unwrap.go @@ -31,12 +31,17 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/watch" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/storage" storageerrors "k8s.io/apiserver/pkg/storage/errors" clientgocache "k8s.io/client-go/tools/cache" + "github.com/kcp-dev/logicalcluster/v3" + + "github.com/kcp-dev/kcp/pkg/cache/client/shard" cachedresourcesreplication "github.com/kcp-dev/kcp/pkg/reconciler/cache/cachedresources/replication" "github.com/kcp-dev/kcp/pkg/tombstone" dynamiccontext "github.com/kcp-dev/kcp/pkg/virtual/framework/dynamic/context" @@ -47,16 +52,48 @@ import ( kcpinformers "github.com/kcp-dev/kcp/sdk/client/informers/externalversions" ) +func unwrapCachedObjectWithCluster(obj *cachev1alpha1.CachedObject, cluster logicalcluster.Name) (*unstructured.Unstructured, error) { + inner, err := unwrapCachedObject(obj) + if err != nil { + return nil, err + } + setCluster(inner, cluster) + + return inner, nil +} + func unwrapCachedObject(obj *cachev1alpha1.CachedObject) (*unstructured.Unstructured, error) { inner := &unstructured.Unstructured{} if err := inner.UnmarshalJSON(obj.Spec.Raw.Raw); err != nil { return nil, fmt.Errorf("failed to decode inner object: %w", err) } - inner.SetResourceVersion(obj.GetResourceVersion()) + + // Set the original UID and RV. + ann := inner.GetAnnotations() + inner.SetUID(types.UID(ann[cachedresourcesreplication.AnnotationKeyOriginalResourceUID])) + inner.SetResourceVersion(ann[cachedresourcesreplication.AnnotationKeyOriginalResourceVersion]) + + // Remove internal annotations from the object before exposing it to the client. + if ann := inner.GetAnnotations(); ann != nil { + delete(ann, shard.AnnotationKey) + delete(ann, cachedresourcesreplication.AnnotationKeyOriginalResourceUID) + delete(ann, cachedresourcesreplication.AnnotationKeyOriginalResourceVersion) + inner.SetAnnotations(ann) + } + return inner, nil } -func withUnwrapping(apiResourceSchema *apisv1alpha1.APIResourceSchema, version string, cacheKcpInformers kcpinformers.SharedInformerFactory) forwardingregistry.StorageWrapper { +func setCluster(obj *unstructured.Unstructured, cluster logicalcluster.Name) { + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + annotations[logicalcluster.AnnotationKey] = cluster.String() + obj.SetAnnotations(annotations) +} + +func withUnwrapping(apiResourceSchema *apisv1alpha1.APIResourceSchema, version string, localKcpInformers, globalKcpInformers kcpinformers.SharedInformerFactory, cr *cachev1alpha1.CachedResource) forwardingregistry.StorageWrapper { wrappedGVR := schema.GroupVersionResource{ Group: apiResourceSchema.Spec.Group, Version: version, @@ -71,14 +108,28 @@ func withUnwrapping(apiResourceSchema *apisv1alpha1.APIResourceSchema, version s if err != nil { return nil, fmt.Errorf("invalid API domain key: %v", err) } + targetCluster := genericapirequest.ClusterFrom(ctx) + if targetCluster.Wildcard { + // ??? + panic("wildcard in get request") + } + + bindings, err := listAPIBindingsByCachedResource(cr.Status.IdentityHash, wrappedGVR.GroupResource(), globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), localKcpInformers.Apis().V1alpha2().APIBindings()) + if err != nil { + return nil, fmt.Errorf("internal error: %v", err) + } + clustersForBindings := listClustersInBindings(bindings) + if !clustersForBindings.Has(targetCluster.Name) { + return nil, apierrors.NewNotFound(wrappedGVR.GroupResource(), name) + } cachedObjName := cachedresourcesreplication.GenCachedObjectName(wrappedGVR, genericapirequest.NamespaceValue(ctx), name) - cachedObj, err := cacheKcpInformers.Cache().V1alpha1().CachedObjects().Cluster(parsedKey.CachedResourceCluster).Lister().Get(cachedObjName) + cachedObj, err := globalKcpInformers.Cache().V1alpha1().CachedObjects().Cluster(parsedKey.CachedResourceCluster).Lister().Get(cachedObjName) if err != nil { return nil, fmt.Errorf("failed to get CachedObject %s for resource %s %s: %v", cachedObjName, wrappedGVR, name, err) } - return unwrapCachedObject(cachedObj) + return unwrapCachedObjectWithCluster(cachedObj, targetCluster.Name) } storage.WatcherFunc = func(ctx context.Context, options *metainternalversion.ListOptions) (watch.Interface, error) { parsedKey, err := apidomainkey.Parse(dynamiccontext.APIDomainKeyFrom(ctx)) @@ -101,7 +152,13 @@ func withUnwrapping(apiResourceSchema *apisv1alpha1.APIResourceSchema, version s } return newUnwrappingWatch(ctx, innerGVR, options, namespaced, genericapirequest.NamespaceValue(ctx), - cacheKcpInformers.Cache().V1alpha1().CachedObjects().Cluster(parsedKey.CachedResourceCluster).Informer()) + globalKcpInformers.Cache().V1alpha1().CachedObjects().Cluster(parsedKey.CachedResourceCluster).Informer(), syntheticClustersProvider( + *genericapirequest.ClusterFrom(ctx), + cr.Status.IdentityHash, + wrappedGVR.GroupResource(), + localKcpInformers, + globalKcpInformers, + )) } storage.ListerFunc = func(ctx context.Context, options *metainternalversion.ListOptions) (runtime.Object, error) { parsedKey, err := apidomainkey.Parse(dynamiccontext.APIDomainKeyFrom(ctx)) @@ -133,7 +190,7 @@ func withUnwrapping(apiResourceSchema *apisv1alpha1.APIResourceSchema, version s innerListGVK.Kind = apiResourceSchema.Spec.Names.Kind + "List" } - cachedObjs, err := cacheKcpInformers.Cache().V1alpha1().CachedObjects().Informer().GetIndexer().ByIndex( + cachedObjs, err := globalKcpInformers.Cache().V1alpha1().CachedObjects().Informer().GetIndexer().ByIndex( cachedresourcesreplication.ByGVRAndLogicalClusterAndNamespace, cachedresourcesreplication.GVRAndLogicalClusterAndNamespace( innerGVR, @@ -145,7 +202,13 @@ func withUnwrapping(apiResourceSchema *apisv1alpha1.APIResourceSchema, version s return nil, err } - return newUnwrappingList(innerListGVK, innerGVR.GroupResource(), cachedObjs, options, namespaced) + return newUnwrappingList(innerListGVK, innerGVR.GroupResource(), cachedObjs, options, namespaced, genericapirequest.NamespaceValue(ctx), syntheticClustersProvider( + *genericapirequest.ClusterFrom(ctx), + cr.Status.IdentityHash, + wrappedGVR.GroupResource(), + localKcpInformers, + globalKcpInformers, + )) } }) } @@ -184,7 +247,7 @@ func checkCrossNamespaceAndWildcard(ctx context.Context, gvr schema.GroupVersion } type unwrappingWatch struct { - lock sync.Mutex + stopLock sync.Mutex // Don't use this anywhere but in Stop()! doneChan chan struct{} resultChan chan watch.Event @@ -199,6 +262,7 @@ func newUnwrappingWatch( namespaced bool, namespace string, scopedCachedObjectsInformer clientgocache.SharedIndexInformer, + syntheticClusters func() []logicalcluster.Name, ) (*unwrappingWatch, error) { w := &unwrappingWatch{ doneChan: make(chan struct{}), @@ -228,7 +292,7 @@ func newUnwrappingWatch( field = innerListOpts.FieldSelector } attrFunc := storage.DefaultClusterScopedAttr - if namespaced { + if namespaced && namespace != "" { attrFunc = storage.DefaultNamespaceScopedAttr } unwrapWithMatchingSelectors := func(cachedObj *cachev1alpha1.CachedObject) (*unstructured.Unstructured, error) { @@ -255,77 +319,100 @@ func newUnwrappingWatch( if cachedObj.GetLabels() == nil { return false } - return cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectGroup] == innerObjGVR.Group && + valid := cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectGroup] == innerObjGVR.Group && cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectVersion] == innerObjGVR.Version && - cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectResource] == innerObjGVR.Resource && - cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectOriginalNamespace] == namespace + cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectResource] == innerObjGVR.Resource + if namespaced && namespace != "" { + valid = valid && cachedObj.Labels[cachedresourcesreplication.LabelKeyObjectOriginalNamespace] == namespace + } + + return valid }, Handler: clientgocache.ResourceEventHandlerDetailedFuncs{ AddFunc: func(obj interface{}, isInInitialList bool) { cachedObj := tombstone.Obj[*cachev1alpha1.CachedObject](obj) if isInInitialList { - if innerListOpts.SendInitialEvents == nil || !*innerListOpts.SendInitialEvents { - // The user explicitly requests to not send the initial list. + if cachedObj.GetResourceVersion() <= innerListOpts.ResourceVersion { + // This resource is older than the want we want to start from on isInInitial list replay. return } - if cachedObj.GetResourceVersion() < innerListOpts.ResourceVersion { - // This resource is older than the want we want to start from on isInInitial list replay. + if innerListOpts.SendInitialEvents != nil && !*innerListOpts.SendInitialEvents { + // The user explicitly requests not to send the initial list. return } } innerObj, err := unwrapWithMatchingSelectors(cachedObj) if err != nil { - w.resultChan <- watch.Event{ + w.safeWrite(watch.Event{ Type: watch.Error, Object: &apierrors.NewInternalError(err).ErrStatus, - } + }) return } if innerObj == nil { // No match because of selectors. return } - w.resultChan <- watch.Event{ - Type: watch.Added, - Object: innerObj, + + for _, cluster := range syntheticClusters() { + obj := innerObj.DeepCopy() + setCluster(obj, cluster) + if !w.safeWrite(watch.Event{ + Type: watch.Added, + Object: obj, + }) { + return + } } }, UpdateFunc: func(oldObj, newObj interface{}) { cachedObj := tombstone.Obj[*cachev1alpha1.CachedObject](newObj) innerObj, err := unwrapWithMatchingSelectors(cachedObj) if err != nil { - w.resultChan <- watch.Event{ + w.safeWrite(watch.Event{ Type: watch.Error, Object: &apierrors.NewInternalError(err).ErrStatus, - } + }) return } if innerObj == nil { return } - w.resultChan <- watch.Event{ - Type: watch.Modified, - Object: innerObj, + for _, cluster := range syntheticClusters() { + obj := innerObj.DeepCopy() + setCluster(obj, cluster) + if !w.safeWrite(watch.Event{ + Type: watch.Modified, + Object: obj, + }) { + return + } } }, DeleteFunc: func(obj interface{}) { cachedObj := tombstone.Obj[*cachev1alpha1.CachedObject](obj) innerObj, err := unwrapWithMatchingSelectors(cachedObj) if err != nil { - w.resultChan <- watch.Event{ + w.safeWrite(watch.Event{ Type: watch.Error, Object: &apierrors.NewInternalError(err).ErrStatus, - } + }) return } if innerObj == nil { // No match because of selectors. return } - w.resultChan <- watch.Event{ - Type: watch.Deleted, - Object: innerObj, + for _, cluster := range syntheticClusters() { + obj := innerObj.DeepCopy() + setCluster(obj, cluster) + if !w.safeWrite(watch.Event{ + Type: watch.Deleted, + Object: obj, + }) { + return + } } }, }, @@ -338,9 +425,23 @@ func newUnwrappingWatch( return w, nil } +// safeWrite sends data to resultChan only if Stop() hasn't been called already. +// It may happen that Stop() is called while watch events are still being sent +// to clusters (when using a wildcard). The CachedObject handler is removed, the +// resultChan closed, but writing to the channel may still continue, causing a panic. +func (w *unwrappingWatch) safeWrite(e watch.Event) bool { + select { + case <-w.doneChan: + return false + default: + w.resultChan <- e + return true + } +} + func (w *unwrappingWatch) Stop() { - w.lock.Lock() - defer w.lock.Unlock() + w.stopLock.Lock() + defer w.stopLock.Unlock() select { case <-w.doneChan: @@ -355,7 +456,7 @@ func (w *unwrappingWatch) ResultChan() <-chan watch.Event { return w.resultChan } -func newUnwrappingList(innerListGVK schema.GroupVersionKind, innerObjGR schema.GroupResource, cachedObjs []interface{}, innerListOpts *metainternalversion.ListOptions, namespaced bool) (*unstructured.UnstructuredList, error) { +func newUnwrappingList(innerListGVK schema.GroupVersionKind, innerObjGR schema.GroupResource, cachedObjs []interface{}, innerListOpts *metainternalversion.ListOptions, namespaced bool, namespace string, syntheticClusters func() []logicalcluster.Name) (*unstructured.UnstructuredList, error) { innerList := &unstructured.UnstructuredList{} innerList.SetGroupVersionKind(innerListGVK) @@ -368,10 +469,16 @@ func newUnwrappingList(innerListGVK schema.GroupVersionKind, innerObjGR schema.G field = innerListOpts.FieldSelector } attrFunc := storage.DefaultClusterScopedAttr - if namespaced { + if namespaced && namespace != "" { attrFunc = storage.DefaultNamespaceScopedAttr } + clusters := syntheticClusters() + + if len(clusters) == 0 { + return innerList, nil + } + latestResourceVersion := "0" for i := range cachedObjs { @@ -393,7 +500,11 @@ func newUnwrappingList(innerListGVK schema.GroupVersionKind, innerObjGR schema.G } innerObj.SetResourceVersion(item.GetResourceVersion()) - innerList.Items = append(innerList.Items, *innerObj) + for _, cluster := range clusters { + obj := innerObj.DeepCopy() + setCluster(obj, cluster) + innerList.Items = append(innerList.Items, *obj) + } if innerObj.GetResourceVersion() > latestResourceVersion { latestResourceVersion = innerObj.GetResourceVersion() @@ -404,3 +515,26 @@ func newUnwrappingList(innerListGVK schema.GroupVersionKind, innerObjGR schema.G return innerList, nil } + +func syntheticClustersProvider(targetCluster genericapirequest.Cluster, identityHash string, wrappedGR schema.GroupResource, localKcpInformers, globalKcpInformers kcpinformers.SharedInformerFactory) func() []logicalcluster.Name { + return func() []logicalcluster.Name { + if targetCluster.Wildcard { + bindings, err := listAPIBindingsByCachedResource(identityHash, wrappedGR, globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), localKcpInformers.Apis().V1alpha2().APIBindings()) + if err != nil { + return nil + } + clustersForBindings := listClustersInBindings(bindings) + return sets.List(clustersForBindings) + } else { + bindings, err := listAPIBindingsByCachedResource(identityHash, wrappedGR, globalKcpInformers.Apis().V1alpha2().APIExports().Informer().GetIndexer(), localKcpInformers.Apis().V1alpha2().APIBindings()) + if err != nil { + return nil + } + clustersForBindings := listClustersInBindings(bindings) + if clustersForBindings.Has(targetCluster.Name) { + return []logicalcluster.Name{targetCluster.Name} + } + return nil + } + } +} diff --git a/sdk/apis/apis/v1alpha1/types_apibinding.go b/sdk/apis/apis/v1alpha1/types_apibinding.go index 04bb84a8f62..80730dc3e27 100644 --- a/sdk/apis/apis/v1alpha1/types_apibinding.go +++ b/sdk/apis/apis/v1alpha1/types_apibinding.go @@ -245,6 +245,9 @@ const ( // for the request. This data is synthetic; it is not stored in etcd and instead is only applied when retrieving // CRs for the CRD. AnnotationAPIIdentityKey = "apis.kcp.io/identity" + // AnnotationSchemaStorageKey is the annotation key for identifying schema storage of an exported resource. + // This data is synthetic; it is not stored in etcd and instead is only applied when retrieving CRs for the CRD. + AnnotationSchemaStorageKey = "apis.kcp.io/schema-storage" ) // BoundAPIResource describes a bound GroupVersionResource through an APIResourceSchema of an APIExport.. diff --git a/sdk/apis/apis/v1alpha2/types_apiexport.go b/sdk/apis/apis/v1alpha2/types_apiexport.go index 684b36bf0ec..9f6d04a6b3e 100644 --- a/sdk/apis/apis/v1alpha2/types_apiexport.go +++ b/sdk/apis/apis/v1alpha2/types_apiexport.go @@ -172,20 +172,48 @@ type ResourceSchema struct { } // ResourceSchemaStorage defines how the resource is stored. +// +// +kubebuilder:validation:XValidation:rule="has(self.crd) != has(self.virtual)",message="Exactly one of crd or virtual must be set" type ResourceSchemaStorage struct { // CRD storage defines that this APIResourceSchema is exposed as // CustomResourceDefinitions inside the workspaces that bind to the APIExport. // Like in vanilla Kubernetes, users can then create, update and delete // custom resources. + // + // +optional CRD *ResourceSchemaStorageCRD `json:"crd,omitempty"` + + // Virtual storage defines that this APIResourceSchema is exposed as + // a projection of the referenced resource inside the workspaces that + // bind to the APIExport. + // + // +optional + Virtual *ResourceSchemaStorageVirtual `json:"virtual,omitempty"` } type ResourceSchemaStorageCRD struct{} +// ResourceSchemaStorageVirtual refers to an endpoint slice object +// from which the virtual resource is served. type ResourceSchemaStorageVirtual struct { // Reference points to another object that has a URL to a virtual workspace // in a "url" field in its status. The object can be of any kind. - Reference corev1.TypedLocalObjectReference `json:"reference"` + // Reference corev1.TypedLocalObjectReference `json:"reference"` + + // Group is the API group of the endpoint slice resource. + Group string `json:"group"` + // Version is the API version of the endpoint slice resource. + Version string `json:"version"` + // Resource is the plural name of the endpoint slice resource. + Resource string `json:"resource"` + + // Name is the name of the endpoint slice object. + Name string `json:"name"` + // IdentityHash is the identity of the virtual resource. + IdentityHash string `json:"identityHash"` + + // Resource selector TBD. + // We are not sure if it belongs here. } // Identity defines the identity of an APIExport, i.e. determines the etcd prefix diff --git a/sdk/apis/apis/v1alpha2/types_apiexport_conversion_test.go b/sdk/apis/apis/v1alpha2/types_apiexport_conversion_test.go index 6e923cd5634..bd5ba5b2c93 100644 --- a/sdk/apis/apis/v1alpha2/types_apiexport_conversion_test.go +++ b/sdk/apis/apis/v1alpha2/types_apiexport_conversion_test.go @@ -21,7 +21,9 @@ import ( "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" ) @@ -92,6 +94,31 @@ func TestConvertV1Alpha2APIExports(t *testing.T) { }}, }, }, + { + Spec: APIExportSpec{ + Resources: []ResourceSchema{{ + Group: "bar", + Name: "foo", + Schema: "v1.foo.bar", + Storage: ResourceSchemaStorage{ + Virtual: &ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: ptr.To("example.com"), + Kind: "MyEndpointSlice", + Name: "my-virtual-resource", + }, + IdentityHash: "123abc", + }, + }, + }}, + PermissionClaims: []PermissionClaim{{ + GroupResource: GroupResource{ + Resource: "configmaps", + }, + Verbs: []string{"get"}, + }}, + }, + }, } scheme := runtime.NewScheme() diff --git a/sdk/apis/cache/v1alpha1/types_cachedresourceendpointslice.go b/sdk/apis/cache/v1alpha1/types_cachedresourceendpointslice.go index dd723d391f8..89548366ddb 100644 --- a/sdk/apis/cache/v1alpha1/types_cachedresourceendpointslice.go +++ b/sdk/apis/cache/v1alpha1/types_cachedresourceendpointslice.go @@ -18,6 +18,8 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" ) // +crd @@ -53,16 +55,32 @@ type CachedResourceEndpointSliceSpec struct { // +kubebuilder:validation:Required // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="CachedResource reference must not be changed" CachedResource CachedResourceReference `json:"cachedResource"` + + // partition points to a partition that is used for filtering the endpoints + // of the CachedResource part of the slice. + // + // +optional + Partition string `json:"partition,omitempty"` } // CachedResourceEndpointSliceStatus defines the observed state of CachedResourceEndpointSlice. type CachedResourceEndpointSliceStatus struct { + // conditions is a list of conditions that apply to the CachedResourceEndpointSlice. + Conditions conditionsv1alpha1.Conditions `json:"conditions,omitempty"` + // endpoints contains all the URLs of the Replication service. // // +optional // +listType=map // +listMapKey=url CachedResourceEndpoints []CachedResourceEndpoint `json:"endpoints"` + + // shardSelector is the selector used to filter the shards. It is used to filter the shards + // when determining partition scope when deriving the endpoints. This is set by owning shard, + // and is used by follower shards to determine if its inscope or not. + // + // +optional + ShardSelector string `json:"shardSelector,omitempty"` } // Using a struct provides an extension point @@ -77,6 +95,29 @@ type CachedResourceEndpoint struct { URL string `json:"url"` } +func (in *CachedResourceEndpointSlice) GetConditions() conditionsv1alpha1.Conditions { + return in.Status.Conditions +} + +func (in *CachedResourceEndpointSlice) SetConditions(conditions conditionsv1alpha1.Conditions) { + in.Status.Conditions = conditions +} + +// These are valid conditions of CachedResourceEndpointSlice in addition to +// CachedResourceValid and related reasons defined with the APIBinding type. +const ( + // PartitionValid is a condition for CachedResourceEndpointSlice that reflects the validity of the referenced Partition. + PartitionValid conditionsv1alpha1.ConditionType = "PartitionValid" + + // EndpointURLsReady is a condition for CachedResourceEndpointSlice that reflects the readiness of the URLs. + // Deprecated: This condition is deprecated and will be removed in a future release. + CachedResourceEndpointSliceURLsReady conditionsv1alpha1.ConditionType = "EndpointURLsReady" + + // PartitionInvalidReferenceReason is a reason for the PartitionValid condition of CachedResourceEndpointSlice that the + // Partition reference is invalid. + PartitionInvalidReferenceReason = "PartitionInvalidReference" +) + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // CachedResourceEndpointSliceList is a list of CachedResourceEndpointSlice resources. diff --git a/test/e2e/virtualresources/cachedresources/vr_cachedresources_test.go b/test/e2e/virtualresources/cachedresources/vr_cachedresources_test.go new file mode 100644 index 00000000000..daf75a9d27e --- /dev/null +++ b/test/e2e/virtualresources/cachedresources/vr_cachedresources_test.go @@ -0,0 +1,590 @@ +/* +Copyright 2025 The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cachedresources + +import ( + "context" + "fmt" + "maps" + "slices" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + apiextensionshelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + kcpapiextensionsclientset "github.com/kcp-dev/client-go/apiextensions/client" + kcpdynamic "github.com/kcp-dev/client-go/dynamic" + "github.com/kcp-dev/logicalcluster/v3" + + apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" + apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" + cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/core" + tenancy1alpha1 "github.com/kcp-dev/kcp/sdk/apis/tenancy/v1alpha1" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" + kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" + kcptesting "github.com/kcp-dev/kcp/sdk/testing" + kcptestinghelpers "github.com/kcp-dev/kcp/sdk/testing/helpers" + "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest" + wildwestv1alpha1 "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/apis/wildwest/v1alpha1" + wildwestclientset "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/client/clientset/versioned/cluster" + "github.com/kcp-dev/kcp/test/e2e/framework" +) + +func TestCachedResources(t *testing.T) { + t.Parallel() + framework.Suite(t, "control-plane") + + server := kcptesting.SharedKcpServer(t) + + orgPath, _ := kcptesting.NewWorkspaceFixture(t, server, core.RootCluster.Path(), kcptesting.WithType(core.RootCluster.Path(), "organization")) + providerPath, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath) + consumer1Path, consumer1WS := kcptesting.NewWorkspaceFixture(t, server, orgPath) + consumer2Path, consumer2WS := kcptesting.NewWorkspaceFixture(t, server, orgPath) + consumerPaths := sets.New(logicalcluster.Name(consumer1WS.Spec.Cluster).Path(), logicalcluster.Name(consumer2WS.Spec.Cluster).Path()) + consumerWorkspaces := map[logicalcluster.Path]*tenancy1alpha1.Workspace{ + consumer1Path: consumer1WS, + consumer2Path: consumer2WS, + } + + t.Logf("providerPath: %v", providerPath) + t.Logf("consumer1Path: %v", consumer1Path) + t.Logf("consumer2Path: %v", consumer2Path) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + cfg := server.BaseConfig(t) + + // + // Create clients. + // + + kcpClusterClient, err := kcpclientset.NewForConfig(cfg) + require.NoError(t, err, "failed to construct kcp cluster client for server") + + kcpDynClusterClient, err := kcpdynamic.NewForConfig(cfg) + require.NoError(t, err, "failed to construct kcp dynamic cluster client for server") + + kcpApiExtensionClusterClient, err := kcpapiextensionsclientset.NewForConfig(cfg) + require.NoError(t, err) + + kcpCRDClusterClient := kcpApiExtensionClusterClient.ApiextensionsV1().CustomResourceDefinitions() + + wildwestClusterClient, err := wildwestclientset.NewForConfig(rest.CopyConfig(cfg)) + require.NoError(t, err) + + // + // Prepare the provider cluster. + // + + resourceNames := sets.New("cowboys", "sheriffs") + + // Prepare wildwest.dev resources "cowboys" and "sheriffs": + // * Create CRD and generate its associated APIResourceSchema to be used with an APIExport later. + // * Create a CachedResource for that resource and wait until it's ready. + cachedResourceIdentities := make(map[string]string) + for resourceName := range resourceNames { + gr := metav1.GroupResource{ + Group: "wildwest.dev", + Resource: resourceName, + } + + t.Logf("Creating %s.wildwest.dev CRD in %q", resourceName, providerPath) + wildwest.Create(t, providerPath, kcpCRDClusterClient, gr) + + crd := wildwest.CRD(t, gr) + sch, err := apisv1alpha1.CRDToAPIResourceSchema(crd, "today") + require.NoError(t, err) + + t.Logf("Creating %s APIResourceSchema in %q", sch.Name, providerPath) + _, err = kcpClusterClient.Cluster(providerPath).ApisV1alpha1().APIResourceSchemas().Create(ctx, sch, metav1.CreateOptions{}) + require.NoError(t, err) + + // Create a CachedResource for that CR. + t.Logf("Creating CachedResource for %s.v1alpha1.wildwest.dev in %q", resourceName, providerPath) + cachedResource, err := kcpClusterClient.Cluster(providerPath).CacheV1alpha1().CachedResources().Create(ctx, &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: gr.String(), + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource{ + Group: "wildwest.dev", + Version: "v1alpha1", + Resource: resourceName, + }, + }, + }, metav1.CreateOptions{}) + require.NoError(t, err) + kcptestinghelpers.EventuallyCondition(t, func() (conditions.Getter, error) { + cachedResource, err = kcpClusterClient.Cluster(providerPath).CacheV1alpha1().CachedResources().Get(ctx, cachedResource.Name, metav1.GetOptions{}) + return cachedResource, err + }, kcptestinghelpers.Is(cachev1alpha1.ReplicationStarted), fmt.Sprintf("CachedResource %v should become ready", cachedResource.Name)) + + cachedResourceIdentities[resourceName] = cachedResource.Status.IdentityHash + } + + // Create an APIExport for the created CachedResources. + t.Logf("Creating APIExport for wildwest.dev CachedResources in %q", providerPath) + apiExport, err := kcpClusterClient.Cluster(providerPath).ApisV1alpha2().APIExports().Create(ctx, &apisv1alpha2.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cached-wildwest-provider", + }, + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "wildwest.dev", + Name: "cowboys", + Schema: "today.cowboys.wildwest.dev", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: ptr.To(cachev1alpha1.SchemeGroupVersion.Group), + Kind: "CachedResourceEndpointSlice", + Name: "cowboys.wildwest.dev", + }, + IdentityHash: cachedResourceIdentities["cowboys"], + }, + }, + }, + { + Group: "wildwest.dev", + Name: "sheriffs", + Schema: "today.sheriffs.wildwest.dev", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: ptr.To(cachev1alpha1.SchemeGroupVersion.Group), + Kind: "CachedResourceEndpointSlice", + Name: "sheriffs.wildwest.dev", + }, + IdentityHash: cachedResourceIdentities["sheriffs"], + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + // + // Prepare the consumer clusters. + // + + // Bind that APIExport in consumer workspaces and wait until the APIs are available and check that OpenAPI works too. + for consumerPath := range consumerPaths { + t.Logf("Binding %s|%s in %s", providerPath, apiExport.Name, consumerPath) + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err = kcpClusterClient.Cluster(consumerPath).ApisV1alpha2().APIBindings().Create(ctx, &apisv1alpha2.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cached-wildwest", + }, + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: providerPath.String(), + Name: apiExport.Name, + }, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Sprintf("failed to create APIBinding: %v", err) + } + return true, "" + }, wait.ForeverTestTimeout, time.Second*1, "waiting to create apibinding") + + for resourceName := range resourceNames { + t.Logf("Waiting for %s.v1alpha1.wildwest.dev API to appear in %q", resourceName, consumerPath) + kcptestinghelpers.Eventually(t, func() (bool, string) { + groupList, err := kcpClusterClient.Cluster(consumerPath).Discovery().ServerGroups() + if err != nil { + return false, fmt.Sprintf("failed to retrieve APIResourceList from discovery: %v", err) + } + return slices.ContainsFunc(groupList.Groups, func(e metav1.APIGroup) bool { + return e.Name == wildwestv1alpha1.SchemeGroupVersion.Group + }), fmt.Sprintf("wildwest.dev group not found in %q", consumerPath) + }, wait.ForeverTestTimeout, time.Second*1, "waiting for wildwest.dev group in %q", consumerPath) + kcptestinghelpers.Eventually(t, func() (bool, string) { + resourceList, err := kcpClusterClient.Cluster(consumerPath).Discovery().ServerResourcesForGroupVersion("wildwest.dev/v1alpha1") + if err != nil { + return false, fmt.Sprintf("failed to retrieve APIResourceList from discovery: %v", err) + } + return slices.ContainsFunc(resourceList.APIResources, func(e metav1.APIResource) bool { + return e.Name == resourceName + }), fmt.Sprintf("%s.v1alpha1.wildwest.dev API not found in %q", resourceName, consumerPath) + }, wait.ForeverTestTimeout, time.Second*1, "waiting for wildwest.dev group in %q", resourceName, consumerPath) + + t.Logf("Ensure %s.v1alpha1.wildwest.dev API is available in OpenAPIv3 endpoint in %q", resourceName, consumerPath) + paths, err := kcpClusterClient.Cluster(consumerPath).Discovery().OpenAPIV3().Paths() + require.NoError(t, err, "error retrieving OpenAPIv3 paths in %q", consumerPath) + resourcePath := fmt.Sprintf("apis/%s", wildwestv1alpha1.SchemeGroupVersion.String()) + require.Contains(t, paths, resourcePath, "OpenAPIv3 paths should include %q in %q", resourcePath, consumerPath) + } + } + + // Map of Host->{Admissible workspaces}. + // A client for a certain host (e.g. a virtual workspace) may be able to reach only + // workspaces co-located on the same shard. This map holds these associations. + admissibleWorkspaces := make(map[string]sets.Set[logicalcluster.Path]) + // The default `cfg` is configured with the external addr, so both + // workspaces are reachable regardless of which shard they are on. + admissibleWorkspaces[cfg.Host] = sets.New(consumer1Path, consumer2Path) + + // Generate APIExport VW client configs. The consumers could each be in a different + // shard, and so we need to wait until (possibly) both appear in APIExportEndpointSlice's + // endpoint URLs. + apiExportVWClientConfigs := make(map[string]*rest.Config) + for consumerPath, consumerWS := range consumerWorkspaces { + t.Logf("Creating APIExport VW client for consumer in %q", consumerPath) + + var vwURL string + kcptestinghelpers.Eventually(t, func() (bool, string) { + // First, we get the APIExportEndpointSlice, and then we extract the URL that fits the consumer's workspace host, i.e. the shard URL. + + apiExportEndpointSlice, err := kcpClusterClient.ApisV1alpha1().Cluster(providerPath).APIExportEndpointSlices().Get(ctx, apiExport.Name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + + var found bool + vwURL, found, err = framework.VirtualWorkspaceURL(ctx, kcpClusterClient, consumerWS, + framework.ExportVirtualWorkspaceURLs(apiExportEndpointSlice)) + if err != nil { + return false, err.Error() + } + return found, fmt.Sprintf("URL for workspace %q not found in APIExportEndpointSlice %s|%s", consumerPath, providerPath, apiExport.Name) + }, wait.ForeverTestTimeout, time.Second*1, "waiting for three cowboys") + + if _, ok := apiExportVWClientConfigs[vwURL]; !ok { + vwCfg := rest.CopyConfig(cfg) + vwCfg.Host = vwURL + apiExportVWClientConfigs[vwURL] = vwCfg + } + + if _, ok := admissibleWorkspaces[vwURL]; !ok { + admissibleWorkspaces[vwURL] = make(sets.Set[logicalcluster.Path]) + } + admissibleWorkspaces[vwURL].Insert(consumerPath) + } + + // + // Verify. + // + + // Make sure there are no cowboys or sheriffs in either of the consumer workspaces at the beginning. + for consumerPath := range consumerPaths { + t.Logf("Ensure that there are no Cowboys in %q", consumerPath) + cowboysList, err := wildwestClusterClient.Cluster(consumerPath).WildwestV1alpha1().Cowboys("default").List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Empty(t, cowboysList.Items, "there should be no Cowboy objects at this point in time") + + t.Logf("Ensure that there are no Sheriffs in %q", consumerPath) + sheriffsList, err := listSheriffs(ctx, kcpDynClusterClient, logicalcluster.Name(consumerPath.String())) + require.NoError(t, err) + require.Empty(t, sheriffsList.Items, "there should be no Sheriff objects at this point in time") + } + + t.Log("Verify watching and wildcards work") + + resourceCounters := map[string]*int32{ + "cowboys": ptr.To[int32](0), + "sheriffs": ptr.To[int32](0), + } + var watchStopFuncs []func() + defer func() { + for _, stop := range watchStopFuncs { + stop() + } + }() + + // Create a watch for each resource we're interested in (Cowboys and Sheriffs) + // against each APIExport VW endpoint we've found above. + for resourceName, counter := range resourceCounters { + for _, cfg := range apiExportVWClientConfigs { + dynClient, err := kcpdynamic.NewForConfig(cfg) + require.NoError(t, err) + + t.Logf("Creating a wildcard watch for %s against %q", resourceName, cfg.Host) + + w, err := dynClient.Resource(wildwestv1alpha1.SchemeGroupVersion.WithResource(resourceName)). + Watch(ctx, metav1.ListOptions{}) + require.NoError(t, err) + resultCh := w.ResultChan() + + // We'll increment the counter each time we see an Added event. + + go func() { + for { + select { + case e, more := <-resultCh: + if !more { + return + } + if e.Type == watch.Added { + atomic.AddInt32(counter, 1) + } + case <-ctx.Done(): + w.Stop() + return + } + } + }() + + watchStopFuncs = append(watchStopFuncs, w.Stop) + } + } + + // Create one Cowboy and one Sheriff in the provider workspace. + // Eventually we should see two of each in the resourcesCounter, + // i.e. one for each consumer workspace. + t.Logf("Creating a Cowboy in %q", providerPath) + cowboyOne, err := wildwestClusterClient.Cluster(providerPath).WildwestV1alpha1().Cowboys("default").Create(ctx, &wildwestv1alpha1.Cowboy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cowboys-1", + }, + }, metav1.CreateOptions{}) + require.NoError(t, err) + t.Logf("Creating a Sheriff in %q", providerPath) + sheriffOne, err := createSheriff(ctx, kcpDynClusterClient, logicalcluster.Name(providerPath.String()), &wildwestv1alpha1.Sheriff{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sheriffs-1", + }, + }) + require.NoError(t, err) + + // Finally wait for the counters to be updated, and equal to two. + kcptestinghelpers.Eventually(t, func() (bool, string) { + for resourceName, counter := range resourceCounters { + count := atomic.LoadInt32(counter) + if count != 2 { + return false, fmt.Sprintf("expected to see 2 %s from wildcard watches, got %d", resourceName, count) + } else { + t.Logf("Seen 2 %s from wildcard watches", resourceName) + } + } + return true, "" + }, wait.ForeverTestTimeout, time.Second*1, "waiting for two cowboys and two sheriffs") + + t.Log("Stopping watches") + for _, stop := range watchStopFuncs { + // Intentionally run stop() twice. Watches shouldn't panic because of that. + stop() + stop() + } + + // Make sure listing * for Cowboys and Sheriffs returns two items for each. + for resourceName := range resourceNames { + counter := 0 + for _, cfg := range apiExportVWClientConfigs { + dynClient, err := kcpdynamic.NewForConfig(cfg) + require.NoError(t, err) + + t.Logf("Wildcard listing for %s against %q should return two items", resourceName, cfg.Host) + + list, err := dynClient.Resource(wildwestv1alpha1.SchemeGroupVersion.WithResource(resourceName)). + List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + + counter += len(list.Items) + } + + require.Equal(t, 2, counter, "Wildcard listing should return two %s", resourceName) + } + + // Make sure listing and getting resources in a specific cluster works, both through + // APIExport VW and regular workspace, and that it has expected contents. + wildwestResourceNamespaces := map[string]string{ + cowboyOne.Name: cowboyOne.Namespace, + sheriffOne.Name: sheriffOne.Namespace, // Actually, no namespace here -- this is just for the consistency's sake + } + namespaceableResource := func(namespace string, nsRes dynamic.NamespaceableResourceInterface) dynamic.ResourceInterface { + if namespace == "" { + return nsRes + } + return nsRes.Namespace(namespace) + } + // Get all dynamic clients in a single slice so that we can do this all in one go. + cfgs := make([]*rest.Config, 0, len(apiExportVWClientConfigs)) + for _, c := range apiExportVWClientConfigs { + cfgs = append(cfgs, c) + } + dynClients := make(map[string]kcpdynamic.ClusterInterface) + for _, cfg := range append(cfgs, cfg) { + dynClient, err := kcpdynamic.NewForConfig(cfg) + require.NoError(t, err) + dynClients[cfg.Host] = dynClient + } + for resourceName := range resourceNames { + // We'll need these for object comparison below. + cowboyOneNormalized := cowboyOne.DeepCopy() + cowboyOneNormalized.APIVersion = wildwestv1alpha1.SchemeGroupVersion.String() + cowboyOneNormalized.Kind = "Cowboy" + cowboyOneNormalized.Annotations = make(map[string]string) + + sheriffOneNormalized := sheriffOne.DeepCopy() + sheriffOneNormalized.APIVersion = wildwestv1alpha1.SchemeGroupVersion.String() + sheriffOneNormalized.Kind = "Sheriff" + sheriffOneNormalized.Annotations = make(map[string]string) + + for consumerPath, consumerWS := range consumerWorkspaces { + // Set the cluster name to the expected values. + cowboyOneNormalized.Annotations[logicalcluster.AnnotationKey] = consumerWS.Spec.Cluster + sheriffOneNormalized.Annotations[logicalcluster.AnnotationKey] = consumerWS.Spec.Cluster + + cowboyOneNormalizedUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(cowboyOneNormalized) + require.NoError(t, err) + + sheriffOneNormalizedUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sheriffOneNormalized) + require.NoError(t, err) + + wildwestObjsNormalizedUnstructured := map[string]map[string]interface{}{ + cowboyOneNormalized.Name: normalizeUnstructuredMap(cowboyOneNormalizedUnstructured), + sheriffOneNormalized.Name: normalizeUnstructuredMap(sheriffOneNormalizedUnstructured), + } + + for host, dynClient := range dynClients { + if !admissibleWorkspaces[host].Has(consumerPath) { + continue + } + + objName := resourceName + "-1" + objNamespace := wildwestResourceNamespaces[objName] + + t.Logf("Listing %s resources in %q via %q should return one object", resourceName, consumerPath, host) + list, err := namespaceableResource( + objNamespace, + dynClient.Cluster(logicalcluster.NewPath(consumerWS.Spec.Cluster)). + Resource(wildwestv1alpha1.SchemeGroupVersion.WithResource(resourceName))). + List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Equal(t, 1, len(list.Items), "Unexpected number of items in %s list in %q when listing through %q", resourceName, consumerPath, host) + + require.NoError(t, err) + require.EqualValues(t, wildwestObjsNormalizedUnstructured[objName], normalizeUnstructuredMap(list.Items[0].Object)) + + t.Logf("Getting a %s resource named %s in %q via %q should return that object", resourceName, objName, consumerPath, host) + obj, err := namespaceableResource( + objNamespace, + dynClient.Cluster(logicalcluster.NewPath(consumerWS.Spec.Cluster)). + Resource(wildwestv1alpha1.SchemeGroupVersion.WithResource(resourceName))). + Get(ctx, objName, metav1.GetOptions{}) + require.NoError(t, err) + require.EqualValues(t, wildwestObjsNormalizedUnstructured[objName], normalizeUnstructuredMap(obj.Object)) + } + } + } + + // Verify that APIBinding's conflict checker blocks creating Sheriff CRD in consumer1WS. + t.Log("Creating a CRD with conflicting name") + sheriffsCRDConflicting, err := kcpCRDClusterClient.Cluster(consumer1Path).Create(ctx, wildwest.CRD(t, metav1.GroupResource{ + Group: wildwestv1alpha1.SchemeGroupVersion.Group, + Resource: "sheriffs", + }), metav1.CreateOptions{}) + require.NoError(t, err) + kcptestinghelpers.Eventually(t, func() (bool, string) { + sheriffsCRDConflicting, err = kcpApiExtensionClusterClient.Cluster(consumer1Path).ApiextensionsV1().CustomResourceDefinitions().Get(ctx, "sheriffs.wildwest.dev", metav1.GetOptions{}) + if err != nil { + return false, fmt.Sprintf("failed to get CRD: %v", err) + } + return apiextensionshelpers.IsCRDConditionFalse(sheriffsCRDConflicting, apiextensionsv1.NamesAccepted), "the CRD should not be accepted because of names collision" + }, wait.ForeverTestTimeout, time.Second*1, "waiting to create apibinding") +} + +func normalizeUnstructuredMap(origObj map[string]interface{}) map[string]interface{} { + obj := maps.Clone(origObj) + + meta, hasMeta := obj["metadata"].(map[string]interface{}) + if hasMeta { + delete(meta, "creationTimestamp") + delete(meta, "resourceVersion") + delete(meta, "selfLink") + delete(meta, "uid") + delete(meta, "generation") + delete(meta, "managedFields") + + ann, hasAnn := meta["annotations"].(map[string]interface{}) + if hasAnn { + // TODO(gman0): HACK! https://github.com/kcp-dev/kcp/issues/3478 + // Partial metadata objects have this annotation added. + // This will go away once we have full objects. + delete(ann, "kcp.io/original-api-version") + } + } + + // TODO(gman0): HACK! https://github.com/kcp-dev/kcp/issues/3478 + // We need to remove spec and status, because we're getting only metadata for now. + // This will go away once we have full objects. + delete(obj, "spec") + delete(obj, "status") + + return obj +} + +func listSheriffs(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name) (*wildwestv1alpha1.SheriffList, error) { + uList, err := c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + var sheriffs wildwestv1alpha1.SheriffList + err = runtime.DefaultUnstructuredConverter.FromUnstructured(uList.UnstructuredContent(), &sheriffs) + if err != nil { + return nil, err + } + + return &sheriffs, nil +} + +func createSheriff(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name, sheriff *wildwestv1alpha1.Sheriff) (*wildwestv1alpha1.Sheriff, error) { + m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sheriff) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{ + Object: m, + } + u.SetAPIVersion(wildwestv1alpha1.SchemeGroupVersion.String()) + u.SetKind("Sheriff") + + u, err = c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).Create(ctx, u, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + createdSheriff := &wildwestv1alpha1.Sheriff{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, createdSheriff) + return createdSheriff, err +} From c559a43cf31f6ea64a381d83ef1a2f1b77449549 Mon Sep 17 00:00:00 2001 From: Robert Vasek Date: Thu, 2 Oct 2025 19:25:29 +0200 Subject: [PATCH 5/7] Regenerated code On-behalf-of: @SAP robert.vasek@sap.com Signed-off-by: Robert Vasek --- config/crds/apis.kcp.io_apiexports.yaml | 43 +++++ ...e.kcp.io_cachedresourceendpointslices.yaml | 57 +++++++ config/crds/cache.kcp.io_cachedresources.yaml | 36 ----- .../root-phase0/apiexport-cache.kcp.io.yaml | 4 +- ...edresourceendpointslices.cache.kcp.io.yaml | 58 ++++++- ...ceschema-cachedresources.cache.kcp.io.yaml | 38 +---- pkg/openapi/zz_generated.openapi.go | 147 ++++++------------ sdk/apis/apis/v1alpha2/types_apiexport.go | 11 +- .../apis/v1alpha2/zz_generated.deepcopy.go | 5 + .../cache/v1alpha1/zz_generated.deepcopy.go | 70 +-------- .../apis/v1alpha2/resourceschemastorage.go | 11 +- .../v1alpha2/resourceschemastoragevirtual.go | 52 +++++++ .../cache/v1alpha1/apiresourceschemasource.go | 48 ------ .../cachedresourceendpointslicespec.go | 9 ++ .../cachedresourceendpointslicestatus.go | 22 +++ .../v1alpha1/cachedresourceschemasource.go | 48 ------ .../cache/v1alpha1/cachedresourcestatus.go | 17 +- .../cache/v1alpha1/crdschemasource.go | 48 ------ sdk/client/applyconfiguration/utils.go | 8 +- 19 files changed, 320 insertions(+), 412 deletions(-) create mode 100644 sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastoragevirtual.go delete mode 100644 sdk/client/applyconfiguration/cache/v1alpha1/apiresourceschemasource.go delete mode 100644 sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceschemasource.go delete mode 100644 sdk/client/applyconfiguration/cache/v1alpha1/crdschemasource.go diff --git a/config/crds/apis.kcp.io_apiexports.yaml b/config/crds/apis.kcp.io_apiexports.yaml index a3676e1b71f..dfdcf268350 100644 --- a/config/crds/apis.kcp.io_apiexports.yaml +++ b/config/crds/apis.kcp.io_apiexports.yaml @@ -470,6 +470,8 @@ spec: oneOf: - required: - crd + - required: + - virtual properties: crd: description: |- @@ -478,7 +480,48 @@ spec: Like in vanilla Kubernetes, users can then create, update and delete custom resources. type: object + virtual: + description: |- + Virtual storage defines that this APIResourceSchema is exposed as + a projection of the referenced resource inside the workspaces that + bind to the APIExport. + properties: + identityHash: + description: IdentityHash is the identity of the virtual + resource. + type: string + reference: + description: |- + Reference points to another object that has a URL to a virtual workspace + in a "url" field in its status. The object can be of any kind. + properties: + apiGroup: + description: |- + APIGroup is the group for the resource being referenced. + If APIGroup is not specified, the specified Kind must be in the core API group. + For any other third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource being + referenced + type: string + name: + description: Name is the name of resource being + referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + required: + - identityHash + - reference + type: object type: object + x-kubernetes-validations: + - message: Exactly one of crd or virtual must be set + rule: has(self.crd) != has(self.virtual) required: - group - name diff --git a/config/crds/cache.kcp.io_cachedresourceendpointslices.yaml b/config/crds/cache.kcp.io_cachedresourceendpointslices.yaml index 6643c870124..2b2369843c1 100644 --- a/config/crds/cache.kcp.io_cachedresourceendpointslices.yaml +++ b/config/crds/cache.kcp.io_cachedresourceendpointslices.yaml @@ -65,6 +65,11 @@ spec: x-kubernetes-validations: - message: CachedResource reference must not be changed rule: self == oldSelf + partition: + description: |- + partition points to a partition that is used for filtering the endpoints + of the CachedResource part of the slice. + type: string required: - cachedResource type: object @@ -73,6 +78,52 @@ spec: status communicates the observed state: the filtered list of endpoints for the Replication service. properties: + conditions: + description: conditions is a list of conditions that apply to the + CachedResourceEndpointSlice. + items: + description: Condition defines an observation of a object operational + state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array endpoints: description: endpoints contains all the URLs of the Replication service. items: @@ -90,6 +141,12 @@ spec: x-kubernetes-list-map-keys: - url x-kubernetes-list-type: map + shardSelector: + description: |- + shardSelector is the selector used to filter the shards. It is used to filter the shards + when determining partition scope when deriving the endpoints. This is set by owning shard, + and is used by follower shards to determine if its inscope or not. + type: string type: object type: object served: true diff --git a/config/crds/cache.kcp.io_cachedresources.yaml b/config/crds/cache.kcp.io_cachedresources.yaml index 31dc891ec3f..a3546fd8969 100644 --- a/config/crds/cache.kcp.io_cachedresources.yaml +++ b/config/crds/cache.kcp.io_cachedresources.yaml @@ -216,42 +216,6 @@ spec: - cache - local type: object - resourceSchemaSource: - description: ResourceSchemaSource is a reference to the schema object - of the cached resource. - properties: - apiResourceSchema: - description: APIResourceSchema defines an APIResourceSchema as - the source of the schema. - properties: - clusterName: - description: ClusterName is the name of the cluster where - the APIResourceSchema is defined. - minLength: 1 - type: string - name: - description: Name is the APIResourceSchema name. - minLength: 1 - type: string - required: - - clusterName - - name - type: object - crd: - description: CRD defines a CRD as the source of the schema. - properties: - name: - description: Name is the CRD name. - minLength: 1 - type: string - resourceVersion: - description: ResourceVersion is the resource version of the - source CRD object. - type: string - required: - - name - type: object - type: object type: object required: - spec diff --git a/config/root-phase0/apiexport-cache.kcp.io.yaml b/config/root-phase0/apiexport-cache.kcp.io.yaml index 0e8f5a925e9..a34c9b9ea66 100644 --- a/config/root-phase0/apiexport-cache.kcp.io.yaml +++ b/config/root-phase0/apiexport-cache.kcp.io.yaml @@ -7,12 +7,12 @@ spec: resources: - group: cache.kcp.io name: cachedresourceendpointslices - schema: v250604-36e6abf17.cachedresourceendpointslices.cache.kcp.io + schema: v251008-c4825908e.cachedresourceendpointslices.cache.kcp.io storage: crd: {} - group: cache.kcp.io name: cachedresources - schema: v250830-080348246.cachedresources.cache.kcp.io + schema: v250920-de0c8484d.cachedresources.cache.kcp.io storage: crd: {} status: {} diff --git a/config/root-phase0/apiresourceschema-cachedresourceendpointslices.cache.kcp.io.yaml b/config/root-phase0/apiresourceschema-cachedresourceendpointslices.cache.kcp.io.yaml index 5e9e9870cde..d6f73345a6c 100644 --- a/config/root-phase0/apiresourceschema-cachedresourceendpointslices.cache.kcp.io.yaml +++ b/config/root-phase0/apiresourceschema-cachedresourceendpointslices.cache.kcp.io.yaml @@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1 kind: APIResourceSchema metadata: creationTimestamp: null - name: v250604-36e6abf17.cachedresourceendpointslices.cache.kcp.io + name: v251008-c4825908e.cachedresourceendpointslices.cache.kcp.io spec: group: cache.kcp.io names: @@ -62,6 +62,11 @@ spec: x-kubernetes-validations: - message: CachedResource reference must not be changed rule: self == oldSelf + partition: + description: |- + partition points to a partition that is used for filtering the endpoints + of the CachedResource part of the slice. + type: string required: - cachedResource type: object @@ -70,6 +75,51 @@ spec: status communicates the observed state: the filtered list of endpoints for the Replication service. properties: + conditions: + description: conditions is a list of conditions that apply to the CachedResourceEndpointSlice. + items: + description: Condition defines an observation of a object operational + state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array endpoints: description: endpoints contains all the URLs of the Replication service. items: @@ -87,6 +137,12 @@ spec: x-kubernetes-list-map-keys: - url x-kubernetes-list-type: map + shardSelector: + description: |- + shardSelector is the selector used to filter the shards. It is used to filter the shards + when determining partition scope when deriving the endpoints. This is set by owning shard, + and is used by follower shards to determine if its inscope or not. + type: string type: object type: object served: true diff --git a/config/root-phase0/apiresourceschema-cachedresources.cache.kcp.io.yaml b/config/root-phase0/apiresourceschema-cachedresources.cache.kcp.io.yaml index 3036f7a9495..cf059e6440a 100644 --- a/config/root-phase0/apiresourceschema-cachedresources.cache.kcp.io.yaml +++ b/config/root-phase0/apiresourceschema-cachedresources.cache.kcp.io.yaml @@ -2,7 +2,7 @@ apiVersion: apis.kcp.io/v1alpha1 kind: APIResourceSchema metadata: creationTimestamp: null - name: v250830-080348246.cachedresources.cache.kcp.io + name: v250920-de0c8484d.cachedresources.cache.kcp.io spec: group: cache.kcp.io names: @@ -213,42 +213,6 @@ spec: - cache - local type: object - resourceSchemaSource: - description: ResourceSchemaSource is a reference to the schema object - of the cached resource. - properties: - apiResourceSchema: - description: APIResourceSchema defines an APIResourceSchema as the - source of the schema. - properties: - clusterName: - description: ClusterName is the name of the cluster where the - APIResourceSchema is defined. - minLength: 1 - type: string - name: - description: Name is the APIResourceSchema name. - minLength: 1 - type: string - required: - - clusterName - - name - type: object - crd: - description: CRD defines a CRD as the source of the schema. - properties: - name: - description: Name is the CRD name. - minLength: 1 - type: string - resourceVersion: - description: ResourceVersion is the resource version of the - source CRD object. - type: string - required: - - name - type: object - type: object type: object required: - spec diff --git a/pkg/openapi/zz_generated.openapi.go b/pkg/openapi/zz_generated.openapi.go index 377b8130a6c..a1e26395048 100644 --- a/pkg/openapi/zz_generated.openapi.go +++ b/pkg/openapi/zz_generated.openapi.go @@ -92,8 +92,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ResourceSelector": schema_sdk_apis_apis_v1alpha2_ResourceSelector(ref), "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ScopedPermissionClaim": schema_sdk_apis_apis_v1alpha2_ScopedPermissionClaim(ref), "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.VirtualWorkspace": schema_sdk_apis_apis_v1alpha2_VirtualWorkspace(ref), - "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.APIResourceSchemaSource": schema_sdk_apis_cache_v1alpha1_APIResourceSchemaSource(ref), - "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CRDSchemaSource": schema_sdk_apis_cache_v1alpha1_CRDSchemaSource(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedObject": schema_sdk_apis_cache_v1alpha1_CachedObject(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedObjectList": schema_sdk_apis_cache_v1alpha1_CachedObjectList(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedObjectSpec": schema_sdk_apis_cache_v1alpha1_CachedObjectSpec(ref), @@ -105,7 +103,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceEndpointSliceStatus": schema_sdk_apis_cache_v1alpha1_CachedResourceEndpointSliceStatus(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceList": schema_sdk_apis_cache_v1alpha1_CachedResourceList(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceReference": schema_sdk_apis_cache_v1alpha1_CachedResourceReference(ref), - "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceSchemaSource": schema_sdk_apis_cache_v1alpha1_CachedResourceSchemaSource(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceSpec": schema_sdk_apis_cache_v1alpha1_CachedResourceSpec(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceStatus": schema_sdk_apis_cache_v1alpha1_CachedResourceStatus(ref), "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.GroupVersionResource": schema_sdk_apis_cache_v1alpha1_GroupVersionResource(ref), @@ -2808,11 +2805,17 @@ func schema_sdk_apis_apis_v1alpha2_ResourceSchemaStorage(ref common.ReferenceCal Ref: ref("github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ResourceSchemaStorageCRD"), }, }, + "virtual": { + SchemaProps: spec.SchemaProps{ + Description: "Virtual storage defines that this APIResourceSchema is exposed as a projection of the referenced resource inside the workspaces that bind to the APIExport.", + Ref: ref("github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ResourceSchemaStorageVirtual"), + }, + }, }, }, }, Dependencies: []string{ - "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ResourceSchemaStorageCRD"}, + "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ResourceSchemaStorageCRD", "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2.ResourceSchemaStorageVirtual"}, } } @@ -2830,7 +2833,8 @@ func schema_sdk_apis_apis_v1alpha2_ResourceSchemaStorageVirtual(ref common.Refer return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "ResourceSchemaStorageVirtual refers to an endpoint slice object from which the virtual resource is served.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "reference": { SchemaProps: spec.SchemaProps{ @@ -2839,8 +2843,16 @@ func schema_sdk_apis_apis_v1alpha2_ResourceSchemaStorageVirtual(ref common.Refer Ref: ref("k8s.io/api/core/v1.TypedLocalObjectReference"), }, }, + "identityHash": { + SchemaProps: spec.SchemaProps{ + Description: "IdentityHash is the identity of the virtual resource.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, }, - Required: []string{"reference"}, + Required: []string{"reference", "identityHash"}, }, }, Dependencies: []string{ @@ -2959,64 +2971,6 @@ func schema_sdk_apis_apis_v1alpha2_VirtualWorkspace(ref common.ReferenceCallback } } -func schema_sdk_apis_cache_v1alpha1_APIResourceSchemaSource(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "clusterName": { - SchemaProps: spec.SchemaProps{ - Description: "ClusterName is the name of the cluster where the APIResourceSchema is defined.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - "name": { - SchemaProps: spec.SchemaProps{ - Description: "Name is the APIResourceSchema name.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - Required: []string{"clusterName", "name"}, - }, - }, - } -} - -func schema_sdk_apis_cache_v1alpha1_CRDSchemaSource(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "name": { - SchemaProps: spec.SchemaProps{ - Description: "Name is the CRD name.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - "resourceVersion": { - SchemaProps: spec.SchemaProps{ - Description: "ResourceVersion is the resource version of the source CRD object.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - Required: []string{"name"}, - }, - }, - } -} - func schema_sdk_apis_cache_v1alpha1_CachedObject(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -3311,6 +3265,13 @@ func schema_sdk_apis_cache_v1alpha1_CachedResourceEndpointSliceSpec(ref common.R Ref: ref("github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceReference"), }, }, + "partition": { + SchemaProps: spec.SchemaProps{ + Description: "partition points to a partition that is used for filtering the endpoints of the CachedResource part of the slice.", + Type: []string{"string"}, + Format: "", + }, + }, }, Required: []string{"cachedResource"}, }, @@ -3327,6 +3288,20 @@ func schema_sdk_apis_cache_v1alpha1_CachedResourceEndpointSliceStatus(ref common Description: "CachedResourceEndpointSliceStatus defines the observed state of CachedResourceEndpointSlice.", Type: []string{"object"}, Properties: map[string]spec.Schema{ + "conditions": { + SchemaProps: spec.SchemaProps{ + Description: "conditions is a list of conditions that apply to the CachedResourceEndpointSlice.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1.Condition"), + }, + }, + }, + }, + }, "endpoints": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -3349,11 +3324,18 @@ func schema_sdk_apis_cache_v1alpha1_CachedResourceEndpointSliceStatus(ref common }, }, }, + "shardSelector": { + SchemaProps: spec.SchemaProps{ + Description: "shardSelector is the selector used to filter the shards. It is used to filter the shards when determining partition scope when deriving the endpoints. This is set by owning shard, and is used by follower shards to determine if its inscope or not.", + Type: []string{"string"}, + Format: "", + }, + }, }, }, }, Dependencies: []string{ - "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceEndpoint"}, + "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceEndpoint", "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1.Condition"}, } } @@ -3428,33 +3410,6 @@ func schema_sdk_apis_cache_v1alpha1_CachedResourceReference(ref common.Reference } } -func schema_sdk_apis_cache_v1alpha1_CachedResourceSchemaSource(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "CachedResourceSchemaSource describes the source of resource schema. Exactly one field is set.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "apiResourceSchema": { - SchemaProps: spec.SchemaProps{ - Description: "APIResourceSchema defines an APIResourceSchema as the source of the schema.", - Ref: ref("github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.APIResourceSchemaSource"), - }, - }, - "crd": { - SchemaProps: spec.SchemaProps{ - Description: "CRD defines a CRD as the source of the schema.", - Ref: ref("github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CRDSchemaSource"), - }, - }, - }, - }, - }, - Dependencies: []string{ - "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.APIResourceSchemaSource", "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CRDSchemaSource"}, - } -} - func schema_sdk_apis_cache_v1alpha1_CachedResourceSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -3525,12 +3480,6 @@ func schema_sdk_apis_cache_v1alpha1_CachedResourceStatus(ref common.ReferenceCal Ref: ref("github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.ResourceCount"), }, }, - "resourceSchemaSource": { - SchemaProps: spec.SchemaProps{ - Description: "ResourceSchemaSource is a reference to the schema object of the cached resource.", - Ref: ref("github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceSchemaSource"), - }, - }, "phase": { SchemaProps: spec.SchemaProps{ Description: "Phase of the workspace (Initializing, Ready, Unavailable).", @@ -3556,7 +3505,7 @@ func schema_sdk_apis_cache_v1alpha1_CachedResourceStatus(ref common.ReferenceCal }, }, Dependencies: []string{ - "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.CachedResourceSchemaSource", "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.ResourceCount", "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1.Condition"}, + "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1.ResourceCount", "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1.Condition"}, } } diff --git a/sdk/apis/apis/v1alpha2/types_apiexport.go b/sdk/apis/apis/v1alpha2/types_apiexport.go index 9f6d04a6b3e..95cb34b41aa 100644 --- a/sdk/apis/apis/v1alpha2/types_apiexport.go +++ b/sdk/apis/apis/v1alpha2/types_apiexport.go @@ -198,17 +198,8 @@ type ResourceSchemaStorageCRD struct{} type ResourceSchemaStorageVirtual struct { // Reference points to another object that has a URL to a virtual workspace // in a "url" field in its status. The object can be of any kind. - // Reference corev1.TypedLocalObjectReference `json:"reference"` + Reference corev1.TypedLocalObjectReference `json:"reference"` - // Group is the API group of the endpoint slice resource. - Group string `json:"group"` - // Version is the API version of the endpoint slice resource. - Version string `json:"version"` - // Resource is the plural name of the endpoint slice resource. - Resource string `json:"resource"` - - // Name is the name of the endpoint slice object. - Name string `json:"name"` // IdentityHash is the identity of the virtual resource. IdentityHash string `json:"identityHash"` diff --git a/sdk/apis/apis/v1alpha2/zz_generated.deepcopy.go b/sdk/apis/apis/v1alpha2/zz_generated.deepcopy.go index 07eae536465..acefe38abbf 100644 --- a/sdk/apis/apis/v1alpha2/zz_generated.deepcopy.go +++ b/sdk/apis/apis/v1alpha2/zz_generated.deepcopy.go @@ -516,6 +516,11 @@ func (in *ResourceSchemaStorage) DeepCopyInto(out *ResourceSchemaStorage) { *out = new(ResourceSchemaStorageCRD) **out = **in } + if in.Virtual != nil { + in, out := &in.Virtual, &out.Virtual + *out = new(ResourceSchemaStorageVirtual) + (*in).DeepCopyInto(*out) + } return } diff --git a/sdk/apis/cache/v1alpha1/zz_generated.deepcopy.go b/sdk/apis/cache/v1alpha1/zz_generated.deepcopy.go index b19b801a0ce..0d96e5ba1ed 100644 --- a/sdk/apis/cache/v1alpha1/zz_generated.deepcopy.go +++ b/sdk/apis/cache/v1alpha1/zz_generated.deepcopy.go @@ -29,38 +29,6 @@ import ( conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIResourceSchemaSource) DeepCopyInto(out *APIResourceSchemaSource) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIResourceSchemaSource. -func (in *APIResourceSchemaSource) DeepCopy() *APIResourceSchemaSource { - if in == nil { - return nil - } - out := new(APIResourceSchemaSource) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CRDSchemaSource) DeepCopyInto(out *CRDSchemaSource) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRDSchemaSource. -func (in *CRDSchemaSource) DeepCopy() *CRDSchemaSource { - if in == nil { - return nil - } - out := new(CRDSchemaSource) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CachedObject) DeepCopyInto(out *CachedObject) { *out = *in @@ -263,6 +231,13 @@ func (in *CachedResourceEndpointSliceSpec) DeepCopy() *CachedResourceEndpointSli // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CachedResourceEndpointSliceStatus) DeepCopyInto(out *CachedResourceEndpointSliceStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(conditionsv1alpha1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } if in.CachedResourceEndpoints != nil { in, out := &in.CachedResourceEndpoints, &out.CachedResourceEndpoints *out = make([]CachedResourceEndpoint, len(*in)) @@ -330,32 +305,6 @@ func (in *CachedResourceReference) DeepCopy() *CachedResourceReference { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CachedResourceSchemaSource) DeepCopyInto(out *CachedResourceSchemaSource) { - *out = *in - if in.APIResourceSchema != nil { - in, out := &in.APIResourceSchema, &out.APIResourceSchema - *out = new(APIResourceSchemaSource) - **out = **in - } - if in.CRD != nil { - in, out := &in.CRD, &out.CRD - *out = new(CRDSchemaSource) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CachedResourceSchemaSource. -func (in *CachedResourceSchemaSource) DeepCopy() *CachedResourceSchemaSource { - if in == nil { - return nil - } - out := new(CachedResourceSchemaSource) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CachedResourceSpec) DeepCopyInto(out *CachedResourceSpec) { *out = *in @@ -391,11 +340,6 @@ func (in *CachedResourceStatus) DeepCopyInto(out *CachedResourceStatus) { *out = new(ResourceCount) **out = **in } - if in.ResourceSchemaSource != nil { - in, out := &in.ResourceSchemaSource, &out.ResourceSchemaSource - *out = new(CachedResourceSchemaSource) - (*in).DeepCopyInto(*out) - } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(conditionsv1alpha1.Conditions, len(*in)) diff --git a/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastorage.go b/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastorage.go index bf87a291e9a..121ce1c2fd8 100644 --- a/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastorage.go +++ b/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastorage.go @@ -25,7 +25,8 @@ import ( // ResourceSchemaStorageApplyConfiguration represents a declarative configuration of the ResourceSchemaStorage type for use // with apply. type ResourceSchemaStorageApplyConfiguration struct { - CRD *apisv1alpha2.ResourceSchemaStorageCRD `json:"crd,omitempty"` + CRD *apisv1alpha2.ResourceSchemaStorageCRD `json:"crd,omitempty"` + Virtual *ResourceSchemaStorageVirtualApplyConfiguration `json:"virtual,omitempty"` } // ResourceSchemaStorageApplyConfiguration constructs a declarative configuration of the ResourceSchemaStorage type for use with @@ -41,3 +42,11 @@ func (b *ResourceSchemaStorageApplyConfiguration) WithCRD(value apisv1alpha2.Res b.CRD = &value return b } + +// WithVirtual sets the Virtual field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Virtual field is set to the value of the last call. +func (b *ResourceSchemaStorageApplyConfiguration) WithVirtual(value *ResourceSchemaStorageVirtualApplyConfiguration) *ResourceSchemaStorageApplyConfiguration { + b.Virtual = value + return b +} diff --git a/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastoragevirtual.go b/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastoragevirtual.go new file mode 100644 index 00000000000..51c90ae2f4e --- /dev/null +++ b/sdk/client/applyconfiguration/apis/v1alpha2/resourceschemastoragevirtual.go @@ -0,0 +1,52 @@ +/* +Copyright The KCP Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha2 + +import ( + v1 "k8s.io/api/core/v1" +) + +// ResourceSchemaStorageVirtualApplyConfiguration represents a declarative configuration of the ResourceSchemaStorageVirtual type for use +// with apply. +type ResourceSchemaStorageVirtualApplyConfiguration struct { + Reference *v1.TypedLocalObjectReference `json:"reference,omitempty"` + IdentityHash *string `json:"identityHash,omitempty"` +} + +// ResourceSchemaStorageVirtualApplyConfiguration constructs a declarative configuration of the ResourceSchemaStorageVirtual type for use with +// apply. +func ResourceSchemaStorageVirtual() *ResourceSchemaStorageVirtualApplyConfiguration { + return &ResourceSchemaStorageVirtualApplyConfiguration{} +} + +// WithReference sets the Reference field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Reference field is set to the value of the last call. +func (b *ResourceSchemaStorageVirtualApplyConfiguration) WithReference(value v1.TypedLocalObjectReference) *ResourceSchemaStorageVirtualApplyConfiguration { + b.Reference = &value + return b +} + +// WithIdentityHash sets the IdentityHash field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the IdentityHash field is set to the value of the last call. +func (b *ResourceSchemaStorageVirtualApplyConfiguration) WithIdentityHash(value string) *ResourceSchemaStorageVirtualApplyConfiguration { + b.IdentityHash = &value + return b +} diff --git a/sdk/client/applyconfiguration/cache/v1alpha1/apiresourceschemasource.go b/sdk/client/applyconfiguration/cache/v1alpha1/apiresourceschemasource.go deleted file mode 100644 index 566d27823e2..00000000000 --- a/sdk/client/applyconfiguration/cache/v1alpha1/apiresourceschemasource.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// APIResourceSchemaSourceApplyConfiguration represents a declarative configuration of the APIResourceSchemaSource type for use -// with apply. -type APIResourceSchemaSourceApplyConfiguration struct { - ClusterName *string `json:"clusterName,omitempty"` - Name *string `json:"name,omitempty"` -} - -// APIResourceSchemaSourceApplyConfiguration constructs a declarative configuration of the APIResourceSchemaSource type for use with -// apply. -func APIResourceSchemaSource() *APIResourceSchemaSourceApplyConfiguration { - return &APIResourceSchemaSourceApplyConfiguration{} -} - -// WithClusterName sets the ClusterName field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ClusterName field is set to the value of the last call. -func (b *APIResourceSchemaSourceApplyConfiguration) WithClusterName(value string) *APIResourceSchemaSourceApplyConfiguration { - b.ClusterName = &value - return b -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *APIResourceSchemaSourceApplyConfiguration) WithName(value string) *APIResourceSchemaSourceApplyConfiguration { - b.Name = &value - return b -} diff --git a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicespec.go b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicespec.go index 28be0827ab5..3da980ca3fb 100644 --- a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicespec.go +++ b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicespec.go @@ -22,6 +22,7 @@ package v1alpha1 // with apply. type CachedResourceEndpointSliceSpecApplyConfiguration struct { CachedResource *CachedResourceReferenceApplyConfiguration `json:"cachedResource,omitempty"` + Partition *string `json:"partition,omitempty"` } // CachedResourceEndpointSliceSpecApplyConfiguration constructs a declarative configuration of the CachedResourceEndpointSliceSpec type for use with @@ -37,3 +38,11 @@ func (b *CachedResourceEndpointSliceSpecApplyConfiguration) WithCachedResource(v b.CachedResource = value return b } + +// WithPartition sets the Partition field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Partition field is set to the value of the last call. +func (b *CachedResourceEndpointSliceSpecApplyConfiguration) WithPartition(value string) *CachedResourceEndpointSliceSpecApplyConfiguration { + b.Partition = &value + return b +} diff --git a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicestatus.go b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicestatus.go index 9c2be377b02..b7dae40513e 100644 --- a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicestatus.go +++ b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceendpointslicestatus.go @@ -18,10 +18,16 @@ limitations under the License. package v1alpha1 +import ( + conditionsv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/apis/conditions/v1alpha1" +) + // CachedResourceEndpointSliceStatusApplyConfiguration represents a declarative configuration of the CachedResourceEndpointSliceStatus type for use // with apply. type CachedResourceEndpointSliceStatusApplyConfiguration struct { + Conditions *conditionsv1alpha1.Conditions `json:"conditions,omitempty"` CachedResourceEndpoints []CachedResourceEndpointApplyConfiguration `json:"endpoints,omitempty"` + ShardSelector *string `json:"shardSelector,omitempty"` } // CachedResourceEndpointSliceStatusApplyConfiguration constructs a declarative configuration of the CachedResourceEndpointSliceStatus type for use with @@ -30,6 +36,14 @@ func CachedResourceEndpointSliceStatus() *CachedResourceEndpointSliceStatusApply return &CachedResourceEndpointSliceStatusApplyConfiguration{} } +// WithConditions sets the Conditions field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Conditions field is set to the value of the last call. +func (b *CachedResourceEndpointSliceStatusApplyConfiguration) WithConditions(value conditionsv1alpha1.Conditions) *CachedResourceEndpointSliceStatusApplyConfiguration { + b.Conditions = &value + return b +} + // WithCachedResourceEndpoints adds the given value to the CachedResourceEndpoints field in the declarative configuration // and returns the receiver, so that objects can be build by chaining "With" function invocations. // If called multiple times, values provided by each call will be appended to the CachedResourceEndpoints field. @@ -42,3 +56,11 @@ func (b *CachedResourceEndpointSliceStatusApplyConfiguration) WithCachedResource } return b } + +// WithShardSelector sets the ShardSelector field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ShardSelector field is set to the value of the last call. +func (b *CachedResourceEndpointSliceStatusApplyConfiguration) WithShardSelector(value string) *CachedResourceEndpointSliceStatusApplyConfiguration { + b.ShardSelector = &value + return b +} diff --git a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceschemasource.go b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceschemasource.go deleted file mode 100644 index 925fdfa34c6..00000000000 --- a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourceschemasource.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// CachedResourceSchemaSourceApplyConfiguration represents a declarative configuration of the CachedResourceSchemaSource type for use -// with apply. -type CachedResourceSchemaSourceApplyConfiguration struct { - APIResourceSchema *APIResourceSchemaSourceApplyConfiguration `json:"apiResourceSchema,omitempty"` - CRD *CRDSchemaSourceApplyConfiguration `json:"crd,omitempty"` -} - -// CachedResourceSchemaSourceApplyConfiguration constructs a declarative configuration of the CachedResourceSchemaSource type for use with -// apply. -func CachedResourceSchemaSource() *CachedResourceSchemaSourceApplyConfiguration { - return &CachedResourceSchemaSourceApplyConfiguration{} -} - -// WithAPIResourceSchema sets the APIResourceSchema field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the APIResourceSchema field is set to the value of the last call. -func (b *CachedResourceSchemaSourceApplyConfiguration) WithAPIResourceSchema(value *APIResourceSchemaSourceApplyConfiguration) *CachedResourceSchemaSourceApplyConfiguration { - b.APIResourceSchema = value - return b -} - -// WithCRD sets the CRD field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the CRD field is set to the value of the last call. -func (b *CachedResourceSchemaSourceApplyConfiguration) WithCRD(value *CRDSchemaSourceApplyConfiguration) *CachedResourceSchemaSourceApplyConfiguration { - b.CRD = value - return b -} diff --git a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourcestatus.go b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourcestatus.go index 35ced3ec9bb..a1ef8575669 100644 --- a/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourcestatus.go +++ b/sdk/client/applyconfiguration/cache/v1alpha1/cachedresourcestatus.go @@ -26,11 +26,10 @@ import ( // CachedResourceStatusApplyConfiguration represents a declarative configuration of the CachedResourceStatus type for use // with apply. type CachedResourceStatusApplyConfiguration struct { - IdentityHash *string `json:"identityHash,omitempty"` - ResourceCounts *ResourceCountApplyConfiguration `json:"resourceCounts,omitempty"` - ResourceSchemaSource *CachedResourceSchemaSourceApplyConfiguration `json:"resourceSchemaSource,omitempty"` - Phase *cachev1alpha1.CachedResourcePhaseType `json:"phase,omitempty"` - Conditions *conditionsv1alpha1.Conditions `json:"conditions,omitempty"` + IdentityHash *string `json:"identityHash,omitempty"` + ResourceCounts *ResourceCountApplyConfiguration `json:"resourceCounts,omitempty"` + Phase *cachev1alpha1.CachedResourcePhaseType `json:"phase,omitempty"` + Conditions *conditionsv1alpha1.Conditions `json:"conditions,omitempty"` } // CachedResourceStatusApplyConfiguration constructs a declarative configuration of the CachedResourceStatus type for use with @@ -55,14 +54,6 @@ func (b *CachedResourceStatusApplyConfiguration) WithResourceCounts(value *Resou return b } -// WithResourceSchemaSource sets the ResourceSchemaSource field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceSchemaSource field is set to the value of the last call. -func (b *CachedResourceStatusApplyConfiguration) WithResourceSchemaSource(value *CachedResourceSchemaSourceApplyConfiguration) *CachedResourceStatusApplyConfiguration { - b.ResourceSchemaSource = value - return b -} - // WithPhase sets the Phase field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the Phase field is set to the value of the last call. diff --git a/sdk/client/applyconfiguration/cache/v1alpha1/crdschemasource.go b/sdk/client/applyconfiguration/cache/v1alpha1/crdschemasource.go deleted file mode 100644 index d79c2d0dbe4..00000000000 --- a/sdk/client/applyconfiguration/cache/v1alpha1/crdschemasource.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by applyconfiguration-gen. DO NOT EDIT. - -package v1alpha1 - -// CRDSchemaSourceApplyConfiguration represents a declarative configuration of the CRDSchemaSource type for use -// with apply. -type CRDSchemaSourceApplyConfiguration struct { - Name *string `json:"name,omitempty"` - ResourceVersion *string `json:"resourceVersion,omitempty"` -} - -// CRDSchemaSourceApplyConfiguration constructs a declarative configuration of the CRDSchemaSource type for use with -// apply. -func CRDSchemaSource() *CRDSchemaSourceApplyConfiguration { - return &CRDSchemaSourceApplyConfiguration{} -} - -// WithName sets the Name field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the Name field is set to the value of the last call. -func (b *CRDSchemaSourceApplyConfiguration) WithName(value string) *CRDSchemaSourceApplyConfiguration { - b.Name = &value - return b -} - -// WithResourceVersion sets the ResourceVersion field in the declarative configuration to the given value -// and returns the receiver, so that objects can be built by chaining "With" function invocations. -// If called multiple times, the ResourceVersion field is set to the value of the last call. -func (b *CRDSchemaSourceApplyConfiguration) WithResourceVersion(value string) *CRDSchemaSourceApplyConfiguration { - b.ResourceVersion = &value - return b -} diff --git a/sdk/client/applyconfiguration/utils.go b/sdk/client/applyconfiguration/utils.go index 0d69285a4ab..c07452c2761 100644 --- a/sdk/client/applyconfiguration/utils.go +++ b/sdk/client/applyconfiguration/utils.go @@ -148,14 +148,14 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &apisv1alpha2.ResourceSchemaApplyConfiguration{} case v1alpha2.SchemeGroupVersion.WithKind("ResourceSchemaStorage"): return &apisv1alpha2.ResourceSchemaStorageApplyConfiguration{} + case v1alpha2.SchemeGroupVersion.WithKind("ResourceSchemaStorageVirtual"): + return &apisv1alpha2.ResourceSchemaStorageVirtualApplyConfiguration{} case v1alpha2.SchemeGroupVersion.WithKind("ScopedPermissionClaim"): return &apisv1alpha2.ScopedPermissionClaimApplyConfiguration{} case v1alpha2.SchemeGroupVersion.WithKind("VirtualWorkspace"): return &apisv1alpha2.VirtualWorkspaceApplyConfiguration{} // Group=cache.kcp.io, Version=v1alpha1 - case cachev1alpha1.SchemeGroupVersion.WithKind("APIResourceSchemaSource"): - return &applyconfigurationcachev1alpha1.APIResourceSchemaSourceApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("CachedObject"): return &applyconfigurationcachev1alpha1.CachedObjectApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("CachedObjectSpec"): @@ -172,14 +172,10 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &applyconfigurationcachev1alpha1.CachedResourceEndpointSliceStatusApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("CachedResourceReference"): return &applyconfigurationcachev1alpha1.CachedResourceReferenceApplyConfiguration{} - case cachev1alpha1.SchemeGroupVersion.WithKind("CachedResourceSchemaSource"): - return &applyconfigurationcachev1alpha1.CachedResourceSchemaSourceApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("CachedResourceSpec"): return &applyconfigurationcachev1alpha1.CachedResourceSpecApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("CachedResourceStatus"): return &applyconfigurationcachev1alpha1.CachedResourceStatusApplyConfiguration{} - case cachev1alpha1.SchemeGroupVersion.WithKind("CRDSchemaSource"): - return &applyconfigurationcachev1alpha1.CRDSchemaSourceApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("GroupVersionResource"): return &applyconfigurationcachev1alpha1.GroupVersionResourceApplyConfiguration{} case cachev1alpha1.SchemeGroupVersion.WithKind("Identity"): From 6d805a43c009cdccee3e5e50a6e9c68a6f665f4f Mon Sep 17 00:00:00 2001 From: Robert Vasek Date: Fri, 3 Oct 2025 03:15:16 +0200 Subject: [PATCH 6/7] e2e: virtual/replication: reworked to fit in with virtual resources On-behalf-of: @SAP robert.vasek@sap.com Signed-off-by: Robert Vasek --- .../apiresourceschema_cowboys_versions.yaml | 81 --- test/e2e/virtual/replication/support.go | 44 -- .../replication/virtualworkspace_test.go | 608 +++++++++--------- 3 files changed, 295 insertions(+), 438 deletions(-) delete mode 100644 test/e2e/virtual/replication/apiresourceschema_cowboys_versions.yaml delete mode 100644 test/e2e/virtual/replication/support.go diff --git a/test/e2e/virtual/replication/apiresourceschema_cowboys_versions.yaml b/test/e2e/virtual/replication/apiresourceschema_cowboys_versions.yaml deleted file mode 100644 index 104ac0a462d..00000000000 --- a/test/e2e/virtual/replication/apiresourceschema_cowboys_versions.yaml +++ /dev/null @@ -1,81 +0,0 @@ -apiVersion: apis.kcp.io/v1alpha1 -kind: APIResourceSchema -metadata: - name: today.cowboys.wildwest.dev -spec: - group: wildwest.dev - names: - kind: Cowboy - listKind: CowboyList - plural: cowboys - singular: cowboy - scope: Namespaced - conversion: - strategy: None - versions: - - name: v1alpha1 - schema: - description: Cowboy is part of the wild west - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CowboySpec holds the desired state of the Cowboy. - properties: - intent: - type: string - type: object - status: - description: CowboyStatus communicates the observed state of the Cowboy. - properties: - result: - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} - - name: v1alpha2 - schema: - description: Cowboy is part of the wild west - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: CowboySpec holds the desired state of the Cowboy. - properties: - intent: - type: string - type: object - status: - description: CowboyStatus communicates the observed state of the Cowboy. - properties: - result: - type: string - type: object - type: object - served: true - storage: false - subresources: - status: {} diff --git a/test/e2e/virtual/replication/support.go b/test/e2e/virtual/replication/support.go deleted file mode 100644 index 1f1675536b0..00000000000 --- a/test/e2e/virtual/replication/support.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2025 The KCP Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package replication - -import ( - "embed" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -//go:embed *.yaml -var testFiles embed.FS - -func groupExists(list *metav1.APIGroupList, group string) bool { - for _, g := range list.Groups { - if g.Name == group { - return true - } - } - return false -} - -func resourceExists(list *metav1.APIResourceList, resource string) bool { - for _, r := range list.APIResources { - if r.Name == resource { - return true - } - } - return false -} diff --git a/test/e2e/virtual/replication/virtualworkspace_test.go b/test/e2e/virtual/replication/virtualworkspace_test.go index 54a8873c312..49b0789fb49 100644 --- a/test/e2e/virtual/replication/virtualworkspace_test.go +++ b/test/e2e/virtual/replication/virtualworkspace_test.go @@ -19,39 +19,41 @@ package replication import ( "context" "fmt" + "slices" "testing" "time" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" - "k8s.io/client-go/discovery/cached/memory" kubernetesclientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/restmapper" - "k8s.io/kube-openapi/pkg/util/sets" + "k8s.io/utils/ptr" - kcpdiscovery "github.com/kcp-dev/client-go/discovery" + kcpapiextensionsclientset "github.com/kcp-dev/client-go/apiextensions/client" kcpdynamic "github.com/kcp-dev/client-go/dynamic" kcpkubernetesclientset "github.com/kcp-dev/client-go/kubernetes" "github.com/kcp-dev/logicalcluster/v3" - "github.com/kcp-dev/kcp/config/helpers" apisv1alpha1 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha1" apisv1alpha2 "github.com/kcp-dev/kcp/sdk/apis/apis/v1alpha2" cachev1alpha1 "github.com/kcp-dev/kcp/sdk/apis/cache/v1alpha1" "github.com/kcp-dev/kcp/sdk/apis/core" + "github.com/kcp-dev/kcp/sdk/apis/third_party/conditions/util/conditions" kcpclientset "github.com/kcp-dev/kcp/sdk/client/clientset/versioned/cluster" kcptesting "github.com/kcp-dev/kcp/sdk/testing" kcptestinghelpers "github.com/kcp-dev/kcp/sdk/testing/helpers" - "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/apis/wildwest" + "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest" wildwestv1alpha1 "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/apis/wildwest/v1alpha1" - wildwestclientset "github.com/kcp-dev/kcp/test/e2e/fixtures/wildwest/client/clientset/versioned/cluster" "github.com/kcp-dev/kcp/test/e2e/framework" ) @@ -66,203 +68,284 @@ func TestCachedResourceVirtualWorkspace(t *testing.T) { cfg := server.BaseConfig(t) - kcpClients, err := kcpclientset.NewForConfig(cfg) + kcpClusterClient, err := kcpclientset.NewForConfig(cfg) require.NoError(t, err, "failed to construct kcp cluster client for server") - dynamicClusterClient, err := kcpdynamic.NewForConfig(cfg) - require.NoError(t, err, "failed to construct dynamic cluster client for server") + kcpDynClusterClient, err := kcpdynamic.NewForConfig(cfg) + require.NoError(t, err, "failed to construct kcp dynamic cluster client for server") kubeClusterClient, err := kcpkubernetesclientset.NewForConfig(cfg) require.NoError(t, err, "failed to construct kube cluster client for server") - wildwestClusterClient, err := wildwestclientset.NewForConfig(cfg) - require.NoError(t, err, "failed to construct wildwest cluster client for server") + kcpApiExtensionClusterClient, err := kcpapiextensionsclientset.NewForConfig(cfg) + require.NoError(t, err) + + kcpCRDClusterClient := kcpApiExtensionClusterClient.ApiextensionsV1().CustomResourceDefinitions() orgPath, _ := kcptesting.NewWorkspaceFixture(t, server, core.RootCluster.Path(), kcptesting.WithType(core.RootCluster.Path(), "organization")) - serviceProviderPath, _ := kcptesting.NewWorkspaceFixture(t, server, orgPath) + serviceProviderPath, serviceProviderWS := kcptesting.NewWorkspaceFixture(t, server, orgPath) + serviceProviderClusterName := logicalcluster.Name(serviceProviderWS.Spec.Cluster) consumerPath, consumerWorkspace := kcptesting.NewWorkspaceFixture(t, server, orgPath) consumerClusterName := logicalcluster.Name(consumerWorkspace.Spec.Cluster) framework.AdmitWorkspaceAccess(ctx, t, kubeClusterClient, serviceProviderPath, []string{"user-1"}, nil, false) framework.AdmitWorkspaceAccess(ctx, t, kubeClusterClient, consumerPath, []string{"user-1"}, nil, false) - setUpServiceProvider(ctx, t, dynamicClusterClient, kcpClients, true, serviceProviderPath, cfg, nil) - bindConsumerToProvider(ctx, t, consumerPath, serviceProviderPath, kcpClients, cfg) - cowboyName1 := createCowboyInConsumer(ctx, t, consumerPath, wildwestClusterClient, nil) - createCachedResourceAndCachedResourceEndpointSliceInConsumer(ctx, t, consumerPath, kcpClients, "cow") + // + // Prepare the provider cluster. + // - t.Logf("Waiting for APIExport to have a virtual workspace URL for the bound workspace %q", consumerWorkspace.Name) - apiExportVWCfg := rest.CopyConfig(cfg) - kcptestinghelpers.Eventually(t, func() (bool, string) { - apiExportEndpointSlice, err := kcpClients.Cluster(serviceProviderPath).ApisV1alpha1().APIExportEndpointSlices().Get(ctx, "today-cowboys", metav1.GetOptions{}) - require.NoError(t, err) - var found bool - apiExportVWCfg.Host, found, err = framework.VirtualWorkspaceURL(ctx, kcpClients, consumerWorkspace, framework.ExportVirtualWorkspaceURLs(apiExportEndpointSlice)) - require.NoError(t, err) - return found, fmt.Sprintf("waiting for virtual workspace URLs to be available: %v", apiExportEndpointSlice.Status.APIExportEndpoints) - }, wait.ForeverTestTimeout, time.Millisecond*100) - - t.Logf("Verifying that the virtual workspace includes the cowboy resource") - wildwestVCClusterClient, err := wildwestclientset.NewForConfig(apiExportVWCfg) + // Prepare wildwest.dev resource "sheriffs": + // * Create CRD and generate its associated APIResourceSchema to be used with an APIExport later. + // * Create a CachedResource for that resource and wait until it's ready. + + gvr := wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs") + + crd := wildwest.CRD(t, metav1.GroupResource(gvr.GroupResource())) + sch, err := apisv1alpha1.CRDToAPIResourceSchema(crd, "today") require.NoError(t, err) - cowboysProjected, err := wildwestVCClusterClient.WildwestV1alpha1().Cowboys().List(ctx, metav1.ListOptions{}) + t.Logf("Creating Sheriff CRD in %q", serviceProviderPath) + wildwest.Create(t, serviceProviderPath, kcpCRDClusterClient, metav1.GroupResource(gvr.GroupResource())) + t.Logf("Creating a Sheriff in %q", serviceProviderPath) + sheriffOne, err := createSheriff(ctx, kcpDynClusterClient, serviceProviderClusterName, &wildwestv1alpha1.Sheriff{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sheriff-1", + }, + }) require.NoError(t, err) - require.Equal(t, 1, len(cowboysProjected.Items)) - t.Logf("Verify that the virtual workspace includes apibindings") - discoveryVCClusterClient, err := kcpdiscovery.NewForConfig(apiExportVWCfg) + t.Logf("Creating %s APIResourceSchema in %q", sch.Name, serviceProviderPath) + _, err = kcpClusterClient.Cluster(serviceProviderPath).ApisV1alpha1().APIResourceSchemas().Create(ctx, sch, metav1.CreateOptions{}) require.NoError(t, err) - resources, err := discoveryVCClusterClient.ServerResourcesForGroupVersion(apisv1alpha2.SchemeGroupVersion.String()) - require.NoError(t, err, "error retrieving APIExport discovery") - require.True(t, resourceExists(resources, "apibindings"), "missing apibindings") - resources, err = discoveryVCClusterClient.ServerResourcesForGroupVersion(apisv1alpha1.SchemeGroupVersion.String()) - require.NoError(t, err, "error retrieving APIExport discovery") - require.True(t, resourceExists(resources, "apibindings"), "missing apibindings") + // Create a CachedResource for that CR. + t.Logf("Creating CachedResource for %s in %q", gvr, serviceProviderPath) + cachedResource, err := kcpClusterClient.Cluster(serviceProviderPath).CacheV1alpha1().CachedResources().Create(ctx, &cachev1alpha1.CachedResource{ + ObjectMeta: metav1.ObjectMeta{ + Name: gvr.GroupResource().String(), + }, + Spec: cachev1alpha1.CachedResourceSpec{ + GroupVersionResource: cachev1alpha1.GroupVersionResource(gvr), + }, + }, metav1.CreateOptions{}) + require.NoError(t, err) + cachedResourceName := cachedResource.Name + kcptestinghelpers.EventuallyCondition(t, func() (conditions.Getter, error) { + cachedResource, err = kcpClusterClient.Cluster(serviceProviderPath).CacheV1alpha1().CachedResources().Get(ctx, cachedResourceName, metav1.GetOptions{}) + return cachedResource, err + }, kcptestinghelpers.Is(cachev1alpha1.ReplicationStarted), fmt.Sprintf("CachedResource %s should become ready", cachedResourceName)) + + // Create an APIExport for the created CachedResource. + t.Logf("Creating APIExport for Sheriff CachedResource in %q", serviceProviderPath) + apiExport, err := kcpClusterClient.Cluster(serviceProviderPath).ApisV1alpha2().APIExports().Create(ctx, &apisv1alpha2.APIExport{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cached-wildwest-provider", + }, + Spec: apisv1alpha2.APIExportSpec{ + Resources: []apisv1alpha2.ResourceSchema{ + { + Group: "wildwest.dev", + Name: "sheriffs", + Schema: "today.sheriffs.wildwest.dev", + Storage: apisv1alpha2.ResourceSchemaStorage{ + Virtual: &apisv1alpha2.ResourceSchemaStorageVirtual{ + Reference: corev1.TypedLocalObjectReference{ + APIGroup: ptr.To(cachev1alpha1.SchemeGroupVersion.Group), + Kind: "CachedResourceEndpointSlice", + Name: "sheriffs.wildwest.dev", + }, + IdentityHash: cachedResource.Status.IdentityHash, + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}) + require.NoError(t, err) + // Wait for export's identity to be available. We'll need it later. + apiExportName := apiExport.Name + kcptestinghelpers.EventuallyCondition(t, func() (conditions.Getter, error) { + apiExport, err = kcpClusterClient.Cluster(serviceProviderPath).ApisV1alpha2().APIExports().Get(ctx, apiExportName, metav1.GetOptions{}) + return apiExport, err + }, kcptestinghelpers.Is(apisv1alpha2.APIExportIdentityValid), fmt.Sprintf("APIExport %s should have its identity ready", apiExportName)) + + // + // Prepare the consumer cluster. + // + + // Bind that APIExport in consumer workspaces. + t.Logf("Binding %s|%s in %s", serviceProviderPath, apiExport.Name, consumerPath) + kcptestinghelpers.Eventually(t, func() (bool, string) { + _, err = kcpClusterClient.Cluster(consumerPath).ApisV1alpha2().APIBindings().Create(ctx, &apisv1alpha2.APIBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cached-wildwest", + }, + Spec: apisv1alpha2.APIBindingSpec{ + Reference: apisv1alpha2.BindingReference{ + Export: &apisv1alpha2.ExportBindingReference{ + Path: serviceProviderPath.String(), + Name: apiExport.Name, + }, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + return false, fmt.Sprintf("failed to create APIBinding: %v", err) + } + return true, "" + }, wait.ForeverTestTimeout, time.Second*1, "waiting to create apibinding") + // Wait until the APIs are available. + t.Logf("Waiting for %s API to appear in %q", gvr, consumerPath) + kcptestinghelpers.Eventually(t, func() (bool, string) { + groupList, err := kcpClusterClient.Cluster(consumerPath).Discovery().ServerGroups() + if err != nil { + return false, fmt.Sprintf("failed to retrieve APIResourceList from discovery: %v", err) + } + return slices.ContainsFunc(groupList.Groups, func(e metav1.APIGroup) bool { + return e.Name == wildwestv1alpha1.SchemeGroupVersion.Group + }), fmt.Sprintf("wildwest.dev group not found in %q", consumerPath) + }, wait.ForeverTestTimeout, time.Second*1, "waiting for wildwest.dev group in %q", consumerPath) + kcptestinghelpers.Eventually(t, func() (bool, string) { + resourceList, err := kcpClusterClient.Cluster(consumerPath).Discovery().ServerResourcesForGroupVersion("wildwest.dev/v1alpha1") + if err != nil { + return false, fmt.Sprintf("failed to retrieve APIResourceList from discovery: %v", err) + } + return slices.ContainsFunc(resourceList.APIResources, func(e metav1.APIResource) bool { + return e.Name == gvr.Resource + }), fmt.Sprintf("%s API not found in %q", gvr, consumerPath) + }, wait.ForeverTestTimeout, time.Second*1, "waiting for wildwest.dev group in %q", consumerPath) + + // + // Verify. + // - t.Logf("Waiting for CachedResource to have a virtual workspace URL for the consumer workspace %q", consumerWorkspace.Name) + // At this point we should have a CachedResourceEndpointSlice ready with one consumer. + cres, err := kcpClusterClient.Cluster(serviceProviderPath).CacheV1alpha1().CachedResourceEndpointSlices().Get(ctx, cachedResourceName, metav1.GetOptions{}) + require.NoError(t, err) cachedResourceVWCfg := rest.CopyConfig(cfg) - kcptestinghelpers.Eventually(t, func() (bool, string) { - cachedResourceEndpointSlice, err := kcpClients.Cluster(consumerPath).CacheV1alpha1().CachedResourceEndpointSlices(). - Get(ctx, "cowboys", metav1.GetOptions{}) - require.NoError(t, err) - var found bool - cachedResourceVWCfg.Host, found, err = framework.VirtualWorkspaceURL(ctx, kcpClients, consumerWorkspace, - framework.ReplicationVirtualWorkspaceURLs(cachedResourceEndpointSlice)) - require.NoError(t, err) - return found, fmt.Sprintf("waiting for virtual workspace URLs to be available: %v", cachedResourceEndpointSlice.Status.CachedResourceEndpoints) - }, wait.ForeverTestTimeout, time.Millisecond*100) + var found bool + cachedResourceVWCfg.Host, found, err = framework.VirtualWorkspaceURL(ctx, kcpClusterClient, consumerWorkspace, + framework.ReplicationVirtualWorkspaceURLs(cres)) + require.NoError(t, err) + require.True(t, found, "expected to have found a suitable VW url for in %v endpoint slice", cres.Status.CachedResourceEndpoints) + cachedResourceVWCfg.Host += ":" + apiExport.Status.IdentityHash // Required to by virtual resources / Replication VW. + // Construct the VW client config. user1CachedResourceVWCfg := framework.StaticTokenUserConfig("user-1", cachedResourceVWCfg) - wwUser1CachedResourceVWConfig, err := wildwestclientset.NewForConfig(user1CachedResourceVWCfg) + user1CachedResourceDynClient, err := kcpdynamic.NewForConfig(user1CachedResourceVWCfg) require.NoError(t, err) t.Logf("=== get ===") { t.Logf("Verify that user-1 cannot GET") - _, err = wwUser1CachedResourceVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default"). - Get(ctx, "cowboy", metav1.GetOptions{}) + _, err = getSheriff(ctx, user1CachedResourceDynClient, consumerClusterName, sheriffOne.Name) require.True(t, apierrors.IsForbidden(err)) - t.Logf("Give user-1 GET access to the virtual workspace") - admit(t, kubeClusterClient.Cluster(consumerPath), "user-1-vw-get", "user-1", "User", - []string{"get"}, wildwestv1alpha1.SchemeGroupVersion.Group, "cowboys", "") + t.Logf("Give user-1 GET access to the virtual workspace and apiexport content") + admit(t, kubeClusterClient.Cluster(serviceProviderPath), "user-1-apiexport-content-get", "user-1", "User", + []string{"get"}, apisv1alpha2.SchemeGroupVersion.Group, "apiexports/content", "") - t.Logf("Verify that user-1 can now get cowboy") + t.Logf("Verify that user-1 can now get sheriff") kcptestinghelpers.Eventually(t, func() (bool, string) { var err error - _, err = wwUser1CachedResourceVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default"). - Get(ctx, cowboyName1, metav1.GetOptions{}) + _, err = getSheriff(ctx, user1CachedResourceDynClient, consumerClusterName, sheriffOne.Name) if apierrors.IsForbidden(err) { return false, fmt.Sprintf("waiting until rbac cache is primed: %v", err) } if apierrors.IsNotFound(err) { - return false, fmt.Sprintf("waiting until the cowboy is in cache: %v", err) + return false, fmt.Sprintf("waiting until the sheriff is in cache: %v", err) } require.NoError(t, err) return true, "" - }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to list cowboys") + }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to list sheriffs") } - /* - t.Logf("Verify that user-1 can now wildcard list cowboys") - kcptestinghelpers.Eventually(t, func() (bool, string) { - cbs, err := wwUser1APIExportVWConfig.WildwestV1alpha1().Cowboys().List(ctx, metav1.ListOptions{}) - if apierrors.IsForbidden(err) { - return false, fmt.Sprintf("waiting until rbac cache is primed: %v", err) - } - require.NoError(t, err) - require.Len(t, cbs.Items, 1, "expected to find exactly one cowboy") - cowboy = &cbs.Items[0] - return true, "" - }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to list cowboys") - - t.Logf("Verify that user-1 can now list cowboys") - cbs, err := wwUser1APIExportVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default").List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - require.Len(t, cbs.Items, 1, "expected to find exactly one cowboy")*/ - - t.Logf("Create another cowboy CR") - cowboyLabels := map[string]string{"hello": "world"} - cowboyName2 := createCowboyInConsumer(ctx, t, consumerPath, wildwestClusterClient, cowboyLabels) - setLabelsOnCowboyInConsumer(ctx, t, consumerPath, wildwestClusterClient, cowboyName2, cowboyLabels) + sheriffLabels := map[string]string{"hello": "world"} + t.Logf("Creating another Sheriff in %q", serviceProviderPath) + sheriffTwo, err := createSheriff(ctx, kcpDynClusterClient, serviceProviderClusterName, &wildwestv1alpha1.Sheriff{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sheriff-2", + Labels: sheriffLabels, + }, + }) + require.NoError(t, err) t.Logf("=== list ===") + var sherrifList *wildwestv1alpha1.SheriffList { t.Logf("Verify that user-1 cannot LIST") - _, err = wwUser1CachedResourceVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default"). - List(ctx, metav1.ListOptions{}) + _, err = listSheriffs(ctx, user1CachedResourceDynClient, consumerClusterName) require.True(t, apierrors.IsForbidden(err)) t.Logf("Give user-1 LIST access to the virtual workspace") - admit(t, kubeClusterClient.Cluster(consumerPath), "user-1-vw-list", "user-1", "User", - []string{"list"}, wildwestv1alpha1.SchemeGroupVersion.Group, "cowboys", "") + admit(t, kubeClusterClient.Cluster(serviceProviderPath), "user-1-apiexport-content-list", "user-1", "User", + []string{"list"}, apisv1alpha2.SchemeGroupVersion.Group, "apiexports/content", "") - t.Logf("Verify that user-1 can now LIST cowboys") - var cowboyList *wildwestv1alpha1.CowboyList + t.Logf("Verify that user-1 can now LIST sheriffs") kcptestinghelpers.Eventually(t, func() (bool, string) { var err error - cowboyList, err = wwUser1CachedResourceVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default"). - List(ctx, metav1.ListOptions{}) + sherrifList, err = listSheriffs(ctx, user1CachedResourceDynClient, consumerClusterName) if apierrors.IsForbidden(err) { return false, fmt.Sprintf("waiting until rbac cache is primed: %v", err) } - if len(cowboyList.Items) < 2 { - return false, fmt.Sprintf("waiting until there are two items in list, have %d", len(cowboyList.Items)) + if len(sherrifList.Items) < 2 { + return false, fmt.Sprintf("waiting until there are two items in list, have %d", len(sherrifList.Items)) } require.NoError(t, err) return true, "" - }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to list cowboys") - require.Len(t, cowboyList.Items, 2, "exptected to find exactly two cowboys") + }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to list sheriffs") + require.Len(t, sherrifList.Items, 2, "expected to find exactly two sheriffs") - t.Logf("Verify that both of the created cowboys are listed") - cowboyNames := sets.NewString() - for i := range cowboyList.Items { - cowboyNames.Insert(cowboyList.Items[i].Name) + t.Logf("### got LIST sheriffs resourceVersion=%s", sherrifList.ResourceVersion) + + t.Logf("Verify that both of the created sheriffs are listed") + sheriffNames := sets.NewString() + for i := range sherrifList.Items { + sheriffNames.Insert(sherrifList.Items[i].Name) } - require.True(t, cowboyNames.HasAll(cowboyName1, cowboyName2), - "expected the list to contain both cowboys, %s and %s, have %s", cowboyName1, cowboyName2, cowboyNames.List()) + require.True(t, sheriffNames.HasAll(sheriffOne.Name, sheriffTwo.Name), + "expected the list to contain both sheriffs, %s and %s, have %s", sheriffOne.Name, sheriffTwo.Name, sheriffNames.List()) } t.Logf("=== watch ===") { t.Logf("Verify that user-1 cannot WATCH") - _, err = wwUser1CachedResourceVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default"). - Watch(ctx, metav1.ListOptions{}) + _, err = watchSheriffs(ctx, user1CachedResourceDynClient, consumerClusterName, metav1.ListOptions{}) require.True(t, apierrors.IsForbidden(err)) t.Logf("Give user-1 WATCH access to the virtual workspace") - admit(t, kubeClusterClient.Cluster(consumerPath), "user-1-vw-watch", "user-1", "User", - []string{"watch"}, wildwestv1alpha1.SchemeGroupVersion.Group, "cowboys", "") + admit(t, kubeClusterClient.Cluster(serviceProviderPath), "user-1-apiexport-content-watch", "user-1", "User", + []string{"watch"}, apisv1alpha2.SchemeGroupVersion.Group, "apiexports/content", "") - t.Logf("Verify that user-1 can now WATCH cowboys") - var cowboyWatch watch.Interface + t.Logf("Verify that user-1 can now WATCH sheriffs") + var sheriffWatch watch.Interface kcptestinghelpers.Eventually(t, func() (bool, string) { var err error - cowboyWatch, err = wwUser1CachedResourceVWConfig.Cluster(consumerClusterName.Path()).WildwestV1alpha1().Cowboys("default").Watch(ctx, metav1.ListOptions{ - LabelSelector: labels.SelectorFromSet(labels.Set(cowboyLabels)).String(), + sheriffWatch, err = watchSheriffs(ctx, user1CachedResourceDynClient, consumerClusterName, metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(labels.Set(sheriffLabels)).String(), + ResourceVersion: sherrifList.ResourceVersion, // We want to see only changes to existing sheriffs. }) if apierrors.IsForbidden(err) { return false, fmt.Sprintf("waiting until rbac cache is primed: %v", err) } require.NoError(t, err) return true, "" - }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to list cowboys") + }, wait.ForeverTestTimeout, time.Millisecond*100, "expected user-1 to watch sheriffs") - cowboyWatchCh := cowboyWatch.ResultChan() + sheriffWatchCh := sheriffWatch.ResultChan() waitForEvent := func() (watch.Event, bool) { var event watch.Event var more bool kcptestinghelpers.Eventually(t, func() (bool, string) { - event, more = <-cowboyWatchCh + event, more = <-sheriffWatchCh return true, "" - }, time.Second*2, time.Millisecond*100, "expected to get a watch event") + }, wait.ForeverTestTimeout, time.Millisecond*100, "expected to get a watch event") return event, more } checkEvent := func(actualEvent watch.Event, expectedEventType watch.EventType, expectedNext, actualNext bool, - checkCowboy func(cowboy *wildwestv1alpha1.Cowboy), + inspectObj func(obj *unstructured.Unstructured), ) { require.Equal(t, expectedNext, actualNext, "unexpected channel state") if !expectedNext { @@ -272,216 +355,30 @@ func TestCachedResourceVirtualWorkspace(t *testing.T) { require.Equal(t, expectedEventType, actualEvent.Type, "unexpected event type") - if checkCowboy != nil { - cowboy := actualEvent.Object.(*wildwestv1alpha1.Cowboy) - checkCowboy(cowboy) + if inspectObj != nil { + obj := actualEvent.Object.(*unstructured.Unstructured) + inspectObj(obj) } } - t.Logf("Set labels on first cowboy to %v", cowboyLabels) - setLabelsOnCowboyInConsumer(ctx, t, consumerPath, wildwestClusterClient, cowboyName1, cowboyLabels) - t.Logf("Verify that the second watched event is the first cowboy with updated labels %v", cowboyLabels) + t.Logf("Set labels on first sheriff to %v", sheriffLabels) + err = setSheriffLabels(ctx, kcpDynClusterClient, serviceProviderClusterName, sheriffOne.Name, sheriffLabels) + require.NoError(t, err, "expected to set labels on sheriff %s", sheriffOne.Name) + + t.Logf("Verify that the second watched event is the first sheriff with updated labels %v", sheriffLabels) e, next := waitForEvent() - checkEvent(e, watch.Modified, true, next, func(cowboy *wildwestv1alpha1.Cowboy) { - require.Equal(t, cowboyName1, cowboy.Name, "expected to receive the first cowboy") - require.Equal(t, cowboyLabels, cowboy.GetLabels(), "expected the cowboy to have labels defined") + checkEvent(e, watch.Modified, true, next, func(obj *unstructured.Unstructured) { + require.Equal(t, sheriffOne.Name, obj.GetName(), "expected to receive the first sheriff") + require.Equal(t, sheriffLabels, obj.GetLabels(), "expected the sheriff to have labels defined") }) t.Logf("Verify that stopping the watch works") - cowboyWatch.Stop() + sheriffWatch.Stop() e, next = waitForEvent() checkEvent(e, watch.Error, false, next, nil) } } -func setUpServiceProvider(ctx context.Context, t *testing.T, dynamicClusterClient kcpdynamic.ClusterInterface, kcpClients kcpclientset.ClusterInterface, multipleVersions bool, serviceProviderWorkspace logicalcluster.Path, cfg *rest.Config, claims []apisv1alpha2.PermissionClaim) { - t.Helper() - t.Logf("Install today cowboys APIResourceSchema into service provider workspace %q", serviceProviderWorkspace) - - serviceProviderClient, err := kcpclientset.NewForConfig(cfg) - require.NoError(t, err) - - apiResPath := "apiresourceschema_cowboys.yaml" - if multipleVersions { - apiResPath = "apiresourceschema_cowboys_versions.yaml" - } - - mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(serviceProviderClient.Cluster(serviceProviderWorkspace).Discovery())) - err = helpers.CreateResourceFromFS(ctx, dynamicClusterClient.Cluster(serviceProviderWorkspace), mapper, nil, apiResPath, testFiles) - require.NoError(t, err) - - t.Logf("Create an APIExport for it") - cowboysAPIExport := &apisv1alpha2.APIExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "today-cowboys", - }, - Spec: apisv1alpha2.APIExportSpec{ - Resources: []apisv1alpha2.ResourceSchema{ - { - Name: "cowboys", - Group: "wildwest.dev", - Schema: "today.cowboys.wildwest.dev", - Storage: apisv1alpha2.ResourceSchemaStorage{ - CRD: &apisv1alpha2.ResourceSchemaStorageCRD{}, - }, - }, - }, - PermissionClaims: claims, - }, - } - _, err = kcpClients.Cluster(serviceProviderWorkspace).ApisV1alpha2().APIExports().Create(ctx, cowboysAPIExport, metav1.CreateOptions{}) - require.NoError(t, err) -} - -func bindConsumerToProvider(ctx context.Context, t *testing.T, consumerWorkspace logicalcluster.Path, providerPath logicalcluster.Path, kcpClients kcpclientset.ClusterInterface, cfg *rest.Config, permissionClaims ...apisv1alpha2.AcceptablePermissionClaim) { - t.Helper() - t.Logf("Create an APIBinding in consumer workspace %q that points to the today-cowboys export from %q", consumerWorkspace, providerPath) - apiBinding := &apisv1alpha2.APIBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cowboys", - }, - Spec: apisv1alpha2.APIBindingSpec{ - Reference: apisv1alpha2.BindingReference{ - Export: &apisv1alpha2.ExportBindingReference{ - Path: providerPath.String(), - Name: "today-cowboys", - }, - }, - PermissionClaims: permissionClaims, - }, - } - - consumerWsClient, err := kcpclientset.NewForConfig(cfg) - require.NoError(t, err) - - kcptestinghelpers.Eventually(t, func() (bool, string) { - _, err = kcpClients.Cluster(consumerWorkspace).ApisV1alpha2().APIBindings().Create(ctx, apiBinding, metav1.CreateOptions{}) - return err == nil, fmt.Sprintf("Error creating APIBinding: %v", err) - }, wait.ForeverTestTimeout, time.Millisecond*100) - - t.Logf("Make sure %q API group shows up in consumer workspace %q group discovery", wildwest.GroupName, consumerWorkspace) - err = wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, wait.ForeverTestTimeout, true, func(c context.Context) (done bool, err error) { - groups, err := consumerWsClient.Cluster(consumerWorkspace).Discovery().ServerGroups() - if err != nil { - return false, fmt.Errorf("error retrieving consumer workspace %q group discovery: %w", consumerWorkspace, err) - } - return groupExists(groups, wildwest.GroupName), nil - }) - require.NoError(t, err) - t.Logf("Make sure cowboys API resource shows up in consumer workspace %q group version discovery", consumerWorkspace) - resources, err := consumerWsClient.Cluster(consumerWorkspace).Discovery().ServerResourcesForGroupVersion(wildwestv1alpha1.SchemeGroupVersion.String()) - require.NoError(t, err, "error retrieving consumer workspace %q API discovery", consumerWorkspace) - require.True(t, resourceExists(resources, "cowboys"), "consumer workspace %q discovery is missing cowboys resource", consumerWorkspace) -} - -func createCowboyInConsumer(ctx context.Context, t *testing.T, consumer1Workspace logicalcluster.Path, wildwestClusterClient wildwestclientset.ClusterInterface, labels map[string]string) string { - t.Helper() - - cowboyClusterClient := wildwestClusterClient.Cluster(consumer1Workspace).WildwestV1alpha1().Cowboys("default") - t.Logf("Create a cowboy CR in consumer workspace %q", consumer1Workspace) - cowboy, err := cowboyClusterClient.Create(ctx, newCowboy("default", labels), metav1.CreateOptions{}) - require.NoError(t, err, "error creating cowboy in consumer workspace %q", consumer1Workspace) - - return cowboy.Name -} - -func setLabelsOnCowboyInConsumer(ctx context.Context, t *testing.T, consumer1Workspace logicalcluster.Path, wildwestClusterClient wildwestclientset.ClusterInterface, name string, labels map[string]string) { - t.Helper() - - cowboyClusterClient := wildwestClusterClient.Cluster(consumer1Workspace).WildwestV1alpha1().Cowboys("default") - t.Logf("Update a cowboy CR in consumer workspace %q with labels %v", consumer1Workspace, labels) - cowboy, err := cowboyClusterClient.Get(ctx, name, metav1.GetOptions{}) - require.NoError(t, err, "error getting cowboy %s in default namespace in consumer workspace %q", name, consumer1Workspace) - - cowboy.ObjectMeta.SetLabels(labels) - _, err = cowboyClusterClient.Update(ctx, cowboy, metav1.UpdateOptions{}) - require.NoError(t, err, "error updating cowboy %s in default namespace in consumer workspace %q", name, consumer1Workspace) -} - -func createCachedResourceAndCachedResourceEndpointSliceInConsumer(ctx context.Context, t *testing.T, consumer1Workspace logicalcluster.Path, kcpClusterClient kcpclientset.ClusterInterface, cowboyName string) { - t.Helper() - - cachedResourcesClient := kcpClusterClient.Cluster(consumer1Workspace).CacheV1alpha1().CachedResources() - cachedResourceEndpointSlicesClient := kcpClusterClient.Cluster(consumer1Workspace).CacheV1alpha1().CachedResourceEndpointSlices() - - t.Logf("Make sure there are no CachedResourceEndpointSlices in consumer workspace %q at the beginning of the test", consumer1Workspace) - var slices *cachev1alpha1.CachedResourceEndpointSliceList - require.Eventually(t, func() bool { - var err error - slices, err = cachedResourceEndpointSlicesClient.List(ctx, metav1.ListOptions{}) - return err == nil - }, wait.ForeverTestTimeout, 100*time.Millisecond, "expected to be able to list") - require.Zero(t, len(slices.Items), "expected 0 CachedResourceEndpointSlices inside consumer workspace %q", consumer1Workspace) - - t.Logf("Make sure there are no CachedResources in consumer workspace %q at the beginning of the test", consumer1Workspace) - var cachedResources *cachev1alpha1.CachedResourceList - require.Eventually(t, func() bool { - var err error - cachedResources, err = kcpClusterClient.Cluster(consumer1Workspace).CacheV1alpha1().CachedResources().List(ctx, metav1.ListOptions{}) - return err == nil - }, wait.ForeverTestTimeout, 100*time.Millisecond, "expected to be able to list") - require.Zero(t, len(cachedResources.Items), "expected 0 CachedResourceEndpointSlices inside consumer workspace %q", consumer1Workspace) - - t.Logf("Create and wait for a CachedResource to cache Cowboys in consumer workspace %q", consumer1Workspace) - cowboysCachedResource := newCachedResourceForCowboy("cowboys") - _, err := cachedResourcesClient.Create(ctx, cowboysCachedResource, metav1.CreateOptions{}) - require.NoError(t, err, "error creating CachedResource in consumer workspace %q", consumer1Workspace) - require.Eventually(t, func() bool { - var err error - cowboysCachedResource, err = cachedResourcesClient.Get(ctx, "cowboys", metav1.GetOptions{}) - return err == nil && cowboysCachedResource.Status.ResourceCounts != nil && cowboysCachedResource.Status.ResourceCounts.Local > 0 && cowboysCachedResource.Status.ResourceCounts.Cache > 0 - }, wait.ForeverTestTimeout, 100*time.Millisecond, - "expected to have CachedResource with non-zero resource count in consumer workspace %q", consumer1Workspace) - require.EqualValues(t, 1, cowboysCachedResource.Status.ResourceCounts.Local, - "expected to have exactly one Cowboy object in consumer workspace %q to be collected by CachedResource", consumer1Workspace) - require.EqualValues(t, 1, cowboysCachedResource.Status.ResourceCounts.Cache, - "expected to have exactly one CachedObject created for cowboy CachedResource in consumer workspace %q", consumer1Workspace) - - t.Logf("Make sure exactly one CachedResourceEndpointSlice was automatically created as a result of creating a CachedResource in consumer workspace %q", consumer1Workspace) - require.Eventually(t, func() bool { - var err error - slices, err = cachedResourceEndpointSlicesClient.List(ctx, metav1.ListOptions{}) - return err == nil && len(slices.Items) > 0 - }, wait.ForeverTestTimeout, 100*time.Millisecond, "expected to have a non-empty list of CachedResourceEndpointSlices") - require.EqualValues(t, 1, len(slices.Items), "expected exactly one CachedResourceEndpointSlices inside consumer workspace %q", consumer1Workspace) -} - -func newCachedResourceForCowboy(name string) *cachev1alpha1.CachedResource { - return &cachev1alpha1.CachedResource{ - TypeMeta: metav1.TypeMeta{ - APIVersion: cachev1alpha1.SchemeGroupVersion.String(), - Kind: "CachedResource", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: cachev1alpha1.CachedResourceSpec{ - GroupVersionResource: cachev1alpha1.GroupVersionResource{ - Group: wildwestv1alpha1.SchemeGroupVersion.Group, - Version: wildwestv1alpha1.SchemeGroupVersion.Version, - Resource: "cowboys", - }, - }, - } -} - -func newCowboy(namespace string, labels map[string]string) *wildwestv1alpha1.Cowboy { - return &wildwestv1alpha1.Cowboy{ - TypeMeta: metav1.TypeMeta{ - APIVersion: wildwestv1alpha1.SchemeGroupVersion.String(), - Kind: "Cowboy", - }, - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - GenerateName: "cowboy-", - Labels: labels, - }, - Spec: wildwestv1alpha1.CowboySpec{ - Intent: "whoa!", - }, - } -} - func createClusterRoleAndBindings(name, subjectName, subjectKind string, verbs []string, resources ...string) (*rbacv1.ClusterRole, *rbacv1.ClusterRoleBinding) { var total = len(resources) / 3 @@ -538,3 +435,88 @@ func admit(t *testing.T, kubeClusterClient kubernetesclientset.Interface, ruleNa _, err = kubeClusterClient.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) require.NoError(t, err) } + +func createSheriff(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name, sheriff *wildwestv1alpha1.Sheriff) (*wildwestv1alpha1.Sheriff, error) { + m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(sheriff) + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{ + Object: m, + } + u.SetAPIVersion(wildwestv1alpha1.SchemeGroupVersion.String()) + u.SetKind("Sheriff") + + u, err = c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).Create(ctx, u, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + + createdSheriff := &wildwestv1alpha1.Sheriff{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, createdSheriff) + return createdSheriff, err +} + +func listSheriffs(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name) (*wildwestv1alpha1.SheriffList, error) { + uList, err := c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + var sheriffs wildwestv1alpha1.SheriffList + err = runtime.DefaultUnstructuredConverter.FromUnstructured(uList.UnstructuredContent(), &sheriffs) + if err != nil { + return nil, err + } + + return &sheriffs, nil +} + +func watchSheriffs(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name, listOpts metav1.ListOptions) (watch.Interface, error) { + uWatch, err := c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).Watch(ctx, listOpts) + if err != nil { + return nil, err + } + + return uWatch, nil +} + +func getSheriff(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name, name string) (*wildwestv1alpha1.Sheriff, error) { + u, err := c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + var sheriff wildwestv1alpha1.Sheriff + err = runtime.DefaultUnstructuredConverter.FromUnstructured(u.UnstructuredContent(), &sheriff) + if err != nil { + return nil, err + } + + return &sheriff, nil +} + +func setSheriffLabels(ctx context.Context, c kcpdynamic.ClusterInterface, cluster logicalcluster.Name, name string, labels map[string]string) error { + u, err := c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + u.SetLabels(labels) + + _, err = c.Cluster(cluster.Path()).Resource( + wildwestv1alpha1.SchemeGroupVersion.WithResource("sheriffs"), + ).Update(ctx, u, metav1.UpdateOptions{}) + + return err +} From fa0b96ed4331993f9cf8eafb9bfb5c4a7477aee4 Mon Sep 17 00:00:00 2001 From: Mangirdas Judeikis Date: Fri, 3 Oct 2025 11:30:57 +0300 Subject: [PATCH 7/7] Start wiring example for VMs --- config/examples/virtualresources/README.md | 55 ++++++++++++ .../examples/virtualresources/apiexport.yaml | 21 +++++ .../apiresourceschema-instances.yaml | 88 +++++++++++++++++++ .../apiresourceschema-virtualmachine.yaml | 78 ++++++++++++++++ .../cached-resource-instances.yaml | 0 .../crd-instances.yaml | 0 .../instances.yaml | 0 7 files changed, 242 insertions(+) create mode 100644 config/examples/virtualresources/README.md create mode 100644 config/examples/virtualresources/apiexport.yaml create mode 100644 config/examples/virtualresources/apiresourceschema-instances.yaml create mode 100644 config/examples/virtualresources/apiresourceschema-virtualmachine.yaml rename config/examples/{cachedresources => virtualresources}/cached-resource-instances.yaml (100%) rename config/examples/{cachedresources => virtualresources}/crd-instances.yaml (100%) rename config/examples/{cachedresources => virtualresources}/instances.yaml (100%) diff --git a/config/examples/virtualresources/README.md b/config/examples/virtualresources/README.md new file mode 100644 index 00000000000..3159719cfac --- /dev/null +++ b/config/examples/virtualresources/README.md @@ -0,0 +1,55 @@ +# VirtualResources Example + +This example shows usage of VirtualResources, together with CachedResources. +The goal of VirtualResources is to distribute static, read-only resources to multiple clusters +in a scalable way. + +## Setup + +1. Start kcp with sharded setup: + + ```bash + make test-run-sharded-server + ``` + +2. Create a provider workspace, where we will create the resources to be distributed: + + ```bash + export KUBECONFIG=.kcp/admin.kubeconfig + kubectl ws create provider --enter + + kubectl create -f config/examples/virtualresources/crd-instances.yaml + # this this to work we always require apiresource schema to be present + kubectl create -f config/examples/virtualresources/apiresourceschema-instances.yaml + kubectl create -f config/examples/virtualresources/instances.yaml + + # create caching for the resources + kubectl create -f config/examples/virtualresources/cached-resource-instances.yaml + ``` + +3. Create a an APIResourceSchema for actual virtual machines to be distributed, + which will be using instance types. + + ```bash + kubectl create -f config/examples/virtualresources/apiresourceschema-virtualmachine.yaml + ``` + +4. Create an APIExport for the virtual machines: + + ```bash + kubectl create -f config/examples/virtualresources/apiexport.yaml + ``` + +5. Create a consumer workspace, where we will consume the virtual machines: + + ```bash + kubectl ws use :root + kubectl ws create consumer --enter + kubectl kcp bind apiexport root:provider:virtualmachines virtualmachines + ``` + +6. Now check if we can see instances in the consumer workspace: + + ```bash + kubectl get instances.machines.svm.io + ``` \ No newline at end of file diff --git a/config/examples/virtualresources/apiexport.yaml b/config/examples/virtualresources/apiexport.yaml new file mode 100644 index 00000000000..619737e3bcf --- /dev/null +++ b/config/examples/virtualresources/apiexport.yaml @@ -0,0 +1,21 @@ +apiVersion: apis.kcp.io/v1alpha2 +kind: APIExport +metadata: + name: virtualmachines +spec: + resources: + - name: instances + group: machines.svm.io + schema: today.instances.machines.svm.io + storage: + virtual: + reference: + apiGroup: cache.kcp.io + kind: CachedResourceEndpointSlice + name: instances + identityHash: 2857921554ab76ec50f25bf083b7aeb4f7808cd169fd2945b007429f426614ec + - name: virtualmachines + group: machines.svm.io + schema: today.virtualmachines.machines.svm.io + storage: + crd: {} \ No newline at end of file diff --git a/config/examples/virtualresources/apiresourceschema-instances.yaml b/config/examples/virtualresources/apiresourceschema-instances.yaml new file mode 100644 index 00000000000..baf59287d87 --- /dev/null +++ b/config/examples/virtualresources/apiresourceschema-instances.yaml @@ -0,0 +1,88 @@ +apiVersion: apis.kcp.io/v1alpha1 +kind: APIResourceSchema +metadata: + name: today.instances.machines.svm.io +spec: + group: machines.svm.io + names: + kind: Instance + listKind: InstanceList + plural: instances + singular: instance + shortNames: + - inst + scope: Cluster + versions: + - name: v1alpha1 + schema: + description: Instance represents a virtual machine instance + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: InstanceSpec holds the desired state of the Instance. + properties: + instanceType: + description: The type of the instance (e.g. small, medium, large) + type: string + name: + description: The name of the instance + type: string + tier: + description: The tier of the instance (e.g. basic, premium) + type: string + enum: + - basic + - premium + - enterprise + required: + - instanceType + - name + - tier + type: object + status: + description: InstanceStatus communicates the observed state of the Instance. + properties: + phase: + description: The current phase of the instance + type: string + enum: + - Pending + - Running + - Terminated + conditions: + description: Current conditions of the instance + items: + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + format: date-time + reason: + type: string + message: + type: string + required: + - type + - status + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/config/examples/virtualresources/apiresourceschema-virtualmachine.yaml b/config/examples/virtualresources/apiresourceschema-virtualmachine.yaml new file mode 100644 index 00000000000..f593f525483 --- /dev/null +++ b/config/examples/virtualresources/apiresourceschema-virtualmachine.yaml @@ -0,0 +1,78 @@ +apiVersion: apis.kcp.io/v1alpha1 +kind: APIResourceSchema +metadata: + name: today.virtualmachines.machines.svm.io +spec: + group: machines.svm.io + names: + kind: VirtualMachine + listKind: VirtualMachineList + plural: virtualmachines + singular: virtualmachine + scope: Namespaced + versions: + - name: v1alpha1 + schema: + description: VirtualMachine represents a virtual machine instance + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: VirtualMachineSpec holds the desired state of the VirtualMachine. + properties: + instanceRef: + description: Reference to an Instance resource + properties: + name: + description: Name of the Instance resource + type: string + namespace: + description: Namespace of the Instance resource + type: string + required: + - name + type: object + required: + - instanceRef + type: object + status: + description: VirtualMachineStatus communicates the observed state of the VirtualMachine. + properties: + phase: + description: Current phase of the virtual machine + type: string + conditions: + description: Current conditions of the virtual machine + items: + properties: + type: + type: string + status: + type: string + lastTransitionTime: + type: string + reason: + type: string + message: + type: string + required: + - type + - status + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} \ No newline at end of file diff --git a/config/examples/cachedresources/cached-resource-instances.yaml b/config/examples/virtualresources/cached-resource-instances.yaml similarity index 100% rename from config/examples/cachedresources/cached-resource-instances.yaml rename to config/examples/virtualresources/cached-resource-instances.yaml diff --git a/config/examples/cachedresources/crd-instances.yaml b/config/examples/virtualresources/crd-instances.yaml similarity index 100% rename from config/examples/cachedresources/crd-instances.yaml rename to config/examples/virtualresources/crd-instances.yaml diff --git a/config/examples/cachedresources/instances.yaml b/config/examples/virtualresources/instances.yaml similarity index 100% rename from config/examples/cachedresources/instances.yaml rename to config/examples/virtualresources/instances.yaml