diff --git a/Dockerfile b/Dockerfile index b7d1749c9..6aecb4729 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ COPY util/ util/ COPY events/ events/ COPY metrics/ metrics/ COPY cleanup/ cleanup/ +COPY pkg/ pkg/ # Build RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o manager main.go diff --git a/apis/controller/v1alpha1/cluster_types.go b/apis/controller/v1alpha1/cluster_types.go index 8e06fedb9..a2d71b911 100644 --- a/apis/controller/v1alpha1/cluster_types.go +++ b/apis/controller/v1alpha1/cluster_types.go @@ -139,6 +139,30 @@ type ClusterStatus struct { // StorageCapabilities contains auto-detected storage capabilities reported by the worker operator. // Populated only when the worker operator's storage-capability reconciler is active. StorageCapabilities *StorageCapabilities `json:"storageCapabilities,omitempty"` + // ActiveController identifies the hub controller that currently holds leadership. + // Populated only on an Active/Standby HA deployment; absent otherwise, so a + // non-HA worker sees no behaviour change. + ActiveController *ActiveControllerInfo `json:"activeController,omitempty"` +} + +// ActiveControllerInfo describes the hub controller currently holding leadership. +// +// Each hub writes this field about itself, on its own API server, and only while +// it holds leadership. A Standby's copy is populated by the state mirror from the +// Active, so it names the Active rather than itself — which lets a worker watching +// both hubs identify the Active by the rule "trust whichever endpoint is reachable +// and reports an ActiveIdentity matching that endpoint's own identity", without +// needing to know which role either hub currently holds. See ADR #293 Decision 7. +type ActiveControllerInfo struct { + // Endpoint is the API server endpoint of the hub currently holding leadership + Endpoint string `json:"endpoint,omitempty"` + // CABundle is the base64-encoded PEM CA bundle for Endpoint + CABundle string `json:"caBundle,omitempty"` + // ActiveIdentity is the HA identity of the hub that wrote this field about itself + ActiveIdentity string `json:"activeIdentity,omitempty"` + // LastUpdated is the timestamp when this declaration was last written. It gives + // a consumer a deterministic tie-break if both hubs self-declare simultaneously. + LastUpdated metav1.Time `json:"lastUpdated,omitempty"` } // StorageCapabilities holds auto-detected RWX-capable storage classes on the worker cluster. diff --git a/apis/controller/v1alpha1/zz_generated.deepcopy.go b/apis/controller/v1alpha1/zz_generated.deepcopy.go index a7f442c9b..a66f3f9f0 100644 --- a/apis/controller/v1alpha1/zz_generated.deepcopy.go +++ b/apis/controller/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,22 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ActiveControllerInfo) DeepCopyInto(out *ActiveControllerInfo) { + *out = *in + in.LastUpdated.DeepCopyInto(&out.LastUpdated) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ActiveControllerInfo. +func (in *ActiveControllerInfo) DeepCopy() *ActiveControllerInfo { + if in == nil { + return nil + } + out := new(ActiveControllerInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cluster) DeepCopyInto(out *Cluster) { *out = *in @@ -181,6 +197,11 @@ func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { *out = new(StorageCapabilities) (*in).DeepCopyInto(*out) } + if in.ActiveController != nil { + in, out := &in.ActiveController, &out.ActiveController + *out = new(ActiveControllerInfo) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. diff --git a/config/crd/bases/controller.kubeslice.io_clusters.yaml b/config/crd/bases/controller.kubeslice.io_clusters.yaml index e354fd5fc..e79fa0e13 100644 --- a/config/crd/bases/controller.kubeslice.io_clusters.yaml +++ b/config/crd/bases/controller.kubeslice.io_clusters.yaml @@ -133,6 +133,31 @@ spec: format: date-time type: string type: object + activeController: + description: |- + ActiveController identifies the hub controller that currently holds leadership. + Populated only on an Active/Standby HA deployment; absent otherwise, so a + non-HA worker sees no behaviour change. + properties: + activeIdentity: + description: ActiveIdentity is the HA identity of the hub that + wrote this field about itself + type: string + caBundle: + description: CABundle is the base64-encoded PEM CA bundle for + Endpoint + type: string + endpoint: + description: Endpoint is the API server endpoint of the hub currently + holding leadership + type: string + lastUpdated: + description: |- + LastUpdated is the timestamp when this declaration was last written. It gives + a consumer a deterministic tie-break if both hubs self-declare simultaneously. + format: date-time + type: string + type: object clusterHealth: description: ClusterHealth shows the health of the worker cluster properties: diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 840c02058..b36424f1d 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -24,6 +24,19 @@ spec: args: - "--health-probe-bind-address=:8081" - "--metrics-bind-address=127.0.0.1:8080" + # Required for the kube-rbac-proxy sidecar above to be able to scrape + # this manager at all. --metrics-secure defaults to TRUE, which makes + # controller-runtime serve the metrics endpoint over TLS *and* wrap it + # in its own authn/authz filter — while the sidecar is configured with + # "--upstream=http://127.0.0.1:8080/", plain HTTP. Left at the default + # the proxy speaks HTTP to a TLS listener and /metrics is unreachable + # through the only path the Service exposes. + # + # Plain HTTP on loopback is the intended kube-rbac-proxy arrangement, + # not a weakening of it: the endpoint is bound to 127.0.0.1 so nothing + # outside the pod can reach it directly, and the sidecar is what + # terminates TLS on 8443 and authorizes callers by SubjectAccessReview. + - "--metrics-secure=false" - "--leader-elect" - "--log-level=debug" - "--rbac-resource-prefix=kubeslice-rbac" diff --git a/config/events/controller.yaml b/config/events/controller.yaml index 2017d0bc8..03cbd84b2 100644 --- a/config/events/controller.yaml +++ b/config/events/controller.yaml @@ -539,3 +539,39 @@ events: type: Warning reportingController: controller message: Warning - Certificate Creation job Failed + - name: HAMirrorSyncFailed + reason: HAMirrorSyncFailed + action: HAMirrorSync + type: Warning + reportingController: controller + message: Failed to mirror a resource from the Active hub onto the Standby; the syncer will retry. + - name: HAPromotedToActive + reason: PromotedToActive + action: HAPromotion + type: Normal + reportingController: controller + message: This hub was promoted from Standby to Active after the previous Active hub's lease went stale. + - name: HABecameActive + reason: BecameActive + action: HAStartup + type: Normal + reportingController: controller + message: This hub started in Active mode and will hold the HA lease and reconcile. + - name: HABecameStandby + reason: BecameStandby + action: HAStartup + type: Normal + reportingController: controller + message: This hub started in Standby mode; it mirrors the Active hub's state and does not reconcile. + - name: HALeadershipLost + reason: LeadershipLost + action: HALeaseRenewal + type: Warning + reportingController: controller + message: This hub failed to renew its HA lease within the renew deadline and has released leadership; it will not reconcile until it renews again. + - name: HAPromotionAborted + reason: PromotionAborted + action: HAPromotion + type: Warning + reportingController: controller + message: A promotion was considered and refused; see ha_promotions_aborted_total and the controller logs for which guard fired. diff --git a/config/events/events_config_map.yaml b/config/events/events_config_map.yaml index bbb1c4e64..bf7fed339 100644 --- a/config/events/events_config_map.yaml +++ b/config/events/events_config_map.yaml @@ -96,4 +96,10 @@ data: - CertificateJobCreationFailed - CertificatesRenewNow - IllegalVPNKeyRotationConfigDelete - - CertificateJobFailed \ No newline at end of file + - CertificateJobFailed + - HAMirrorSyncFailed + - HAPromotedToActive + - HABecameActive + - HABecameStandby + - HALeadershipLost + - HAPromotionAborted \ No newline at end of file diff --git a/config/ha/README.md b/config/ha/README.md new file mode 100644 index 000000000..064d84654 --- /dev/null +++ b/config/ha/README.md @@ -0,0 +1,59 @@ +# HA cross-cluster RBAC (issue #295) + +`active-cluster-clusterrole.yaml` is a least-privilege grant for the +identity behind a Standby's `--ha-active-kubeconfig` flag: read-only +(`get`/`list`/`watch`) access to `Namespace` plus every resource type in +`pkg/ha.CRDMirrorSet` and `pkg/ha.CredentialMirrorSet` (`RemoteSyncer` +never writes to the Active cluster — only reads), plus the Active's own +`coordination.k8s.io/v1` `Lease` — the same kubeconfig is also used by +#294's `WatchRemoteLease` to read the Active's Lease directly, not just by +`RemoteSyncer` to mirror resources. + +## This is not applied by this repo's own deploy flow + +Nothing here is referenced by `config/rbac/kustomization.yaml` or +`config/default/kustomization.yaml`. Those govern the RBAC a controller +grants *itself* on the cluster it's running in. This manifest is different: +it must be applied **on the Active hub cluster**, granting access to +whatever identity the *Standby's* kubeconfig authenticates as — a cluster +this repo's own kustomize overlays have no way to reach, since it's a +separate cluster entirely. + +Apply it manually (or via whatever provisioning tooling manages the Active +hub) against the Active cluster: + +``` +kubectl --context apply -f active-cluster-clusterrole.yaml +``` + +Fill in the `ClusterRoleBinding`'s `subjects` first — the correct subject +depends on how the Standby authenticates to the Active (a `ServiceAccount` +if dialing in-cluster, a client-cert `User` if using a flattened +kubeconfig Secret, as the current dev demo does). + +## A known, deliberate gap this manifest does not close + +Nothing in this repo automates applying this to a real Active cluster — +that's cross-cluster provisioning, out of scope for a single controller +repo. Logged as a follow-up, not built. + +## Credential mirroring and the Secret-read tradeoff + +`pkg/ha.CredentialMirrorSet` (Secrets with the SA-token type filtered out, +ServiceAccounts, Roles, RoleBindings — for #297's post-promotion use) is +wired in, and this `ClusterRole` grants the reads it needs. Weigh the +Secret rule before applying it: RBAC cannot scope `Secret` access by +`.type` or by namespace *label*, and a `ClusterRole` + +`ClusterRoleBinding` is cluster-wide — so the Standby's identity can read +**every** Secret on the Active hub, not just the gateway-certificate +Secrets `RemoteSyncer` actually mirrors. The syncer itself only *copies* +credential objects whose namespace it also mirrors (the label-scoped +project-namespace boundary — notably excluding the controller's own +namespace, whose name can match the project-namespace prefix) and +excludes SA-token Secrets from the watch entirely, but none of that +narrows what the identity *could* read if the kubeconfig leaked — +protect it like the credential it is. The narrower alternative — per-namespace `RoleBinding`s in each +project namespace instead of the cluster-wide binding — works with the +same `ClusterRole`, at the cost of maintaining those bindings as projects +come and go (cross-cluster provisioning tooling this repo deliberately +does not ship). diff --git a/config/ha/active-cluster-clusterrole.yaml b/config/ha/active-cluster-clusterrole.yaml new file mode 100644 index 000000000..b9c15d09f --- /dev/null +++ b/config/ha/active-cluster-clusterrole.yaml @@ -0,0 +1,113 @@ +--- +# Least-privilege grant for the identity behind the Standby's +# --ha-active-kubeconfig, applied ON THE ACTIVE CLUSTER — not part of this +# repo's own config/rbac (that governs the local cluster's own ClusterRole) +# and not referenced by config/rbac/kustomization.yaml or +# config/default/kustomization.yaml, so it is never auto-applied to the +# Standby's own cluster, where it would be meaningless. +# +# See config/ha/README.md for how and where to apply this. +# +# Scope: read-only (get/list/watch) on Namespace plus every type in +# pkg/ha.CRDMirrorSet and pkg/ha.CredentialMirrorSet, plus the Active's own +# HA Lease — the same --ha-active-kubeconfig identity is also used by #294's +# WatchRemoteLease to read the Active's coordination.k8s.io/v1 Lease +# (checkRemoteLeaseOnce), not just by RemoteSyncer. Confirmed live: without +# this, the Standby can mirror CRDs fine but permanently fails to read the +# Active's Lease, so it can never observe staleness in the first place. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kubeslice-ha-standby-reader +rules: + - apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - apiGroups: + - controller.kubeslice.io + resources: + - projects + - clusters + - sliceconfigs + - serviceexportconfigs + - sliceqosconfigs + - vpnkeyrotations + verbs: + - get + - list + - watch + - apiGroups: + - worker.kubeslice.io + resources: + - workersliceconfigs + - workerslicegateways + - workerserviceimports + verbs: + - get + - list + - watch + # Credential mirroring (pkg/ha.CredentialMirrorSet). SECURITY TRADEOFF, + # named rather than buried: RBAC cannot scope Secret access by .type or by + # namespace *label*, and a ClusterRole+ClusterRoleBinding grants reads + # CLUSTER-WIDE — so this rule lets the Standby's identity read every + # Secret on the Active hub, not just the ones RemoteSyncer actually + # mirrors. The syncer's mirrored-namespace gate narrows what is COPIED, + # never what this identity COULD read; treat the --ha-active-kubeconfig + # credential accordingly. To narrow the grant itself, replace this rule + # with per-namespace RoleBindings in each kubeslice project namespace — at + # the cost of maintaining them as projects come and go, which is + # cross-cluster provisioning tooling this repo deliberately does not ship. + # + # Service-account-token Secrets are within this grant and are read + # deliberately: the Standby mirrors their shells so its own token + # controller can mint a worker credential valid on itself before any + # failover. The token VALUES never leave the Active — they are dropped on + # the way into the Standby's cache and again before any write. + - apiGroups: + - "" + resources: + - secrets + - serviceaccounts + verbs: + - get + - list + - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch +--- +# Template only — the subject is deployment-specific (a ServiceAccount if the +# Standby dials the Active in-cluster, a cert CN if it uses client-cert auth +# via a flattened kubeconfig) and can't be hardcoded here. Fill in `subjects` +# before applying. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubeslice-ha-standby-reader-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubeslice-ha-standby-reader +subjects: + - kind: User # or ServiceAccount — see README.md + name: CHANGEME + apiGroup: rbac.authorization.k8s.io diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 25b95a447..29f6cfd54 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -30,6 +30,14 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - namespaces/status + verbs: + - get + - patch + - update - apiGroups: - "" resources: diff --git a/controllers/controller/cluster_controller.go b/controllers/controller/cluster_controller.go index a6d2c79bc..603065c8c 100644 --- a/controllers/controller/cluster_controller.go +++ b/controllers/controller/cluster_controller.go @@ -18,35 +18,61 @@ package controller import ( "context" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" "go.uber.org/zap" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" ) // ClusterReconciler reconciles a Cluster object type ClusterReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme ClusterService service.IClusterService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // SetupWithManager sets up the controller with the Manager. func (c *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controllerv1alpha1.Cluster{}). - Complete(c) + b := ctrl.NewControllerManagedBy(mgr). + For(&controllerv1alpha1.Cluster{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if c.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(c.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(c) } // Reconcile is a function to reconcile the cluster , ClusterReconciler implements it func (c *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if c.LeaderElector != nil && !c.LeaderElector.IsLeader() { + c.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, c.Client, c.Scheme, "ClusterController", c.EventRecorder) return c.ClusterService.ReconcileCluster(kubeSliceCtx, req) } diff --git a/controllers/controller/leader_gate_test.go b/controllers/controller/leader_gate_test.go new file mode 100644 index 000000000..898d09b3d --- /dev/null +++ b/controllers/controller/leader_gate_test.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/kubeslice/kubeslice-controller/pkg/ha" +) + +// TestReconcile_StandbySkipsAndLogs verifies the HA write fence: a Standby +// controller returns immediately from Reconcile without touching its service +// (left nil here — a leaking gate would panic), and logs the skip message on +// every call, proving IsLeader() is evaluated per invocation. +func TestReconcile_StandbySkipsAndLogs(t *testing.T) { + core, logs := observer.New(zapcore.InfoLevel) + logger := zap.New(core).Sugar() + + standby := ha.NewClusterLeaderElector(nil, nil, ha.Options{ + Mode: ha.ModeStandby, + Log: zap.NewNop().Sugar(), + }) + require.False(t, standby.IsLeader(), "standby must not be leader") + + r := &SliceConfigReconciler{ + Log: logger, + LeaderElector: standby, + // SliceConfigService is intentionally nil: the gate must return before it. + } + + const calls = 3 + for i := 0; i < calls; i++ { + res, err := r.Reconcile(context.Background(), ctrl.Request{}) + require.NoError(t, err) + assert.Equal(t, ctrl.Result{}, res) + } + + assert.Equal(t, calls, logs.FilterMessage("standby mode, skipping reconcile").Len(), + "expected one skip log per Reconcile call") +} diff --git a/controllers/controller/project_controller.go b/controllers/controller/project_controller.go index b82be404f..28b0e5e9f 100644 --- a/controllers/controller/project_controller.go +++ b/controllers/controller/project_controller.go @@ -18,35 +18,61 @@ package controller import ( "context" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" "go.uber.org/zap" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" ) // ProjectReconciler reconciles a Project object type ProjectReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme ProjectService service.IProjectService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // SetupWithManager sets up the controller with the Manager. func (t *ProjectReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controllerv1alpha1.Project{}). - Complete(t) + b := ctrl.NewControllerManagedBy(mgr). + For(&controllerv1alpha1.Project{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if t.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(t.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(t) } // Reconcile is a function to reconcile the project, ProjectReconciler implements it func (t *ProjectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if t.LeaderElector != nil && !t.LeaderElector.IsLeader() { + t.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, t.Client, t.Scheme, "ProjectController", t.EventRecorder) return t.ProjectService.ReconcileProject(kubeSliceCtx, req) } diff --git a/controllers/controller/promotion_kick_test.go b/controllers/controller/promotion_kick_test.go new file mode 100644 index 000000000..8dbfb666b --- /dev/null +++ b/controllers/controller/promotion_kick_test.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/manager" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/util" +) + +func kickTestManager(t *testing.T) manager.Manager { + t.Helper() + s := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(s)) + require.NoError(t, controllerv1alpha1.AddToScheme(s)) + + // The address is never dialled: SetupWithManager only registers watches, it + // does not start the manager. + mgr, err := manager.New(&rest.Config{Host: "127.0.0.1:1"}, manager.Options{ + Scheme: s, + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + require.NoError(t, err) + return mgr +} + +// TestSetupWithManager_NilPromotionKick guards a footgun introduced by wiring +// the HA promotion kick into every reconciler. source.Channel rejects a nil +// channel — "must specify Channel.Source" — when the manager starts the source, +// so registering the watch unconditionally breaks every caller that does not +// wire a kick. The envtest suite in this package is one such caller, and any +// out-of-tree consumer constructing these reconcilers is another. +// +// Nil must mean "no extra watch", which is what makes the field genuinely +// optional and keeps a non-HA deployment identical to before. +func TestSetupWithManager_NilPromotionKick(t *testing.T) { + mgr := kickTestManager(t) + r := &ClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: util.NewLogger().With("name", "test"), + } + require.NoError(t, r.SetupWithManager(mgr), + "a reconciler with no promotion kick must set up cleanly") +} + +func TestSetupWithManager_WithPromotionKick(t *testing.T) { + mgr := kickTestManager(t) + // A different type from the nil case: controller-runtime enforces globally + // unique controller names, so reusing ClusterReconciler here would collide + // with the test above rather than test anything. + r := &ProjectReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Log: util.NewLogger().With("name", "test"), + PromotionKick: make(chan event.GenericEvent, 1), + } + require.NoError(t, r.SetupWithManager(mgr), + "and one with a kick must register the extra watch without error") +} diff --git a/controllers/controller/serviceexportconfig_controller.go b/controllers/controller/serviceexportconfig_controller.go index bba9a975f..05235a1de 100644 --- a/controllers/controller/serviceexportconfig_controller.go +++ b/controllers/controller/serviceexportconfig_controller.go @@ -18,35 +18,61 @@ package controller import ( "context" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" "go.uber.org/zap" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" ) // ServiceExportConfigReconciler reconciles a ServiceExportConfig object type ServiceExportConfigReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme ServiceExportConfigService service.IServiceExportConfigService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // Reconcile is a function to reconcile the ServiceExportConfig, ServiceExportConfigReconciler implements it func (r *ServiceExportConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if r.LeaderElector != nil && !r.LeaderElector.IsLeader() { + r.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "ServiceExportConfigController", r.EventRecorder) return r.ServiceExportConfigService.ReconcileServiceExportConfig(kubeSliceCtx, req) } // SetupWithManager sets up the controller with the Manager. func (r *ServiceExportConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controllerv1alpha1.ServiceExportConfig{}). - Complete(r) + b := ctrl.NewControllerManagedBy(mgr). + For(&controllerv1alpha1.ServiceExportConfig{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if r.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(r.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(r) } diff --git a/controllers/controller/sliceconfig_controller.go b/controllers/controller/sliceconfig_controller.go index 99b4be95a..df317465a 100644 --- a/controllers/controller/sliceconfig_controller.go +++ b/controllers/controller/sliceconfig_controller.go @@ -23,32 +23,57 @@ import ( "go.uber.org/zap" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" ) // SliceConfigReconciler reconciles a SliceConfig object type SliceConfigReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme SliceConfigService service.ISliceConfigService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // Reconcile is a function to reconcile the slice config, SliceConfigReconciler implements it func (r *SliceConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if r.LeaderElector != nil && !r.LeaderElector.IsLeader() { + r.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "SliceConfigController", r.EventRecorder) return r.SliceConfigService.ReconcileSliceConfig(kubeSliceCtx, req) } // SetupWithManager sets up the controller with the Manager. func (r *SliceConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controllerv1alpha1.SliceConfig{}). - Complete(r) + b := ctrl.NewControllerManagedBy(mgr). + For(&controllerv1alpha1.SliceConfig{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if r.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(r.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(r) } diff --git a/controllers/controller/sliceqosconfig_controller.go b/controllers/controller/sliceqosconfig_controller.go index d6f7ca90c..4d2e76056 100644 --- a/controllers/controller/sliceqosconfig_controller.go +++ b/controllers/controller/sliceqosconfig_controller.go @@ -18,6 +18,8 @@ package controller import ( "context" + + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" "github.com/kubeslice/kubeslice-monitoring/pkg/events" @@ -26,28 +28,52 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" ) // SliceQoSConfigReconciler reconciles a SliceQoSConfig object type SliceQoSConfigReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme SliceQoSConfigService service.ISliceQoSConfigService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // SetupWithManager sets up the controller with the Manager. func (r *SliceQoSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controllerv1alpha1.SliceQoSConfig{}). - Complete(r) + b := ctrl.NewControllerManagedBy(mgr). + For(&controllerv1alpha1.SliceQoSConfig{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if r.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(r.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(r) } // Reconcile is a function to reconcile the qos_profile, SliceQoSConfigReconciler implements it func (r *SliceQoSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if r.LeaderElector != nil && !r.LeaderElector.IsLeader() { + r.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "SliceQoSConfigController", r.EventRecorder) return r.SliceQoSConfigService.ReconcileSliceQoSConfig(kubeSliceCtx, req) } diff --git a/controllers/controller/vpnkey_rotation_controller.go b/controllers/controller/vpnkey_rotation_controller.go index b88815605..87d46ce6f 100644 --- a/controllers/controller/vpnkey_rotation_controller.go +++ b/controllers/controller/vpnkey_rotation_controller.go @@ -18,6 +18,8 @@ package controller import ( "context" + + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" "github.com/kubeslice/kubeslice-monitoring/pkg/events" @@ -26,28 +28,52 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" ) // VpnKeyRotationReconciler reconciles a VpnKeyRotation object type VpnKeyRotationReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme VpnKeyRotationService service.IVpnKeyRotationService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // SetupWithManager sets up the controller with the Manager. func (r *VpnKeyRotationReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&controllerv1alpha1.VpnKeyRotation{}). - Complete(r) + b := ctrl.NewControllerManagedBy(mgr). + For(&controllerv1alpha1.VpnKeyRotation{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if r.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(r.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(r) } // Reconcile is a function to reconcile the VpnKeyRotation, VpnKeyRotationReconciler implements it func (r *VpnKeyRotationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if r.LeaderElector != nil && !r.LeaderElector.IsLeader() { + r.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "VpnKeyRotationController", r.EventRecorder) return r.VpnKeyRotationService.ReconcileVpnKeyRotation(kubeSliceCtx, req) } diff --git a/controllers/worker/workerserviceimport_controller.go b/controllers/worker/workerserviceimport_controller.go index 2c8b922de..c3b59439e 100644 --- a/controllers/worker/workerserviceimport_controller.go +++ b/controllers/worker/workerserviceimport_controller.go @@ -18,13 +18,18 @@ package worker import ( "context" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" "go.uber.org/zap" "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -32,22 +37,43 @@ import ( // WorkerServiceImportReconciler reconciles a SliceConfig object type WorkerServiceImportReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme WorkerServiceImportService service.IWorkerServiceImportService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // Reconcile is a function to reconcile the workerServiceImport, WorkerServiceImportReconciler implements it func (r *WorkerServiceImportReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if r.LeaderElector != nil && !r.LeaderElector.IsLeader() { + r.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "WorkerServiceImportController", r.EventRecorder) return r.WorkerServiceImportService.ReconcileWorkerServiceImport(kubeSliceCtx, req) } // SetupWithManager sets up the controller with the Manager. func (r *WorkerServiceImportReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.WorkerServiceImport{}). - Complete(r) + b := ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.WorkerServiceImport{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if r.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(r.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(r) } diff --git a/controllers/worker/workersliceconfig_controller.go b/controllers/worker/workersliceconfig_controller.go index f499f0b61..f60ce5654 100644 --- a/controllers/worker/workersliceconfig_controller.go +++ b/controllers/worker/workersliceconfig_controller.go @@ -18,37 +18,62 @@ package worker import ( "context" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" "go.uber.org/zap" + workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" - workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" - "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" ) // WorkerSliceConfigReconciler reconciles a Cluster object type WorkerSliceConfigReconciler struct { + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent client.Client Scheme *runtime.Scheme WorkerSliceService service.IWorkerSliceConfigService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // SetupWithManager sets up the controller with the Manager. func (c *WorkerSliceConfigReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&workerv1alpha1.WorkerSliceConfig{}). - Complete(c) + b := ctrl.NewControllerManagedBy(mgr). + For(&workerv1alpha1.WorkerSliceConfig{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if c.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(c.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(c) } // Reconcile is a function to reconcilation of WorkerSliceconfig, WorkerSliceConfigReconciler implements it func (c *WorkerSliceConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if c.LeaderElector != nil && !c.LeaderElector.IsLeader() { + c.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, c.Client, c.Scheme, "WorkerSliceConfigController", c.EventRecorder) return c.WorkerSliceService.ReconcileWorkerSliceConfig(kubeSliceCtx, req) } diff --git a/controllers/worker/workerslicegateway_controller.go b/controllers/worker/workerslicegateway_controller.go index 4ec43294e..5cd45d7b1 100644 --- a/controllers/worker/workerslicegateway_controller.go +++ b/controllers/worker/workerslicegateway_controller.go @@ -18,13 +18,18 @@ package worker import ( "context" + "github.com/kubeslice/kubeslice-monitoring/pkg/events" "go.uber.org/zap" "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" "k8s.io/apimachinery/pkg/runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,21 +38,42 @@ import ( // WorkerSliceGatewayReconciler reconciles a SliceConfig object type WorkerSliceGatewayReconciler struct { client.Client + // PromotionKick, when set, delivers one event per existing object after a + // promotion. The HA write fence drops reconcile requests rather than + // requeuing them, so flipping it reconciles nothing that already existed; + // this is what wakes that state up. Nil outside HA, which registers no + // extra watch and leaves behaviour unchanged. + PromotionKick <-chan event.GenericEvent Scheme *runtime.Scheme WorkerSliceGatewayService service.IWorkerSliceGatewayService Log *zap.SugaredLogger EventRecorder *events.EventRecorder + // LeaderElector gates mutating reconciles on cross-cluster leadership. It is + // nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293. + LeaderElector *ha.ClusterLeaderElector } // Reconcile is a function, WorkerSliceGatewayReconciler implements it func (r *WorkerSliceGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + // HA write fence: only the Active hub (or a standalone controller) writes. + // A Standby evaluates this on every call and no-ops. + if r.LeaderElector != nil && !r.LeaderElector.IsLeader() { + r.Log.Info("standby mode, skipping reconcile") + return ctrl.Result{}, nil + } kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "WorkerSliceGatewayController", r.EventRecorder) return r.WorkerSliceGatewayService.ReconcileWorkerSliceGateways(kubeSliceCtx, req) } // SetupWithManager sets up the controller with the Manager. func (r *WorkerSliceGatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&v1alpha1.WorkerSliceGateway{}). - Complete(r) + b := ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.WorkerSliceGateway{}) + // Registered only when set. source.Channel rejects a nil channel when the + // manager starts it, so an unconditional watch would break every caller that + // does not wire the kick — the envtest suite among them. + if r.PromotionKick != nil { + b = b.WatchesRawSource(source.Channel(r.PromotionKick, &handler.EnqueueRequestForObject{})) + } + return b.Complete(r) } diff --git a/docs/ha-runbook.md b/docs/ha-runbook.md new file mode 100644 index 000000000..d7ea3949d --- /dev/null +++ b/docs/ha-runbook.md @@ -0,0 +1,513 @@ +# Active/Standby HA runbook + +Operator procedures for the cross-cluster HA controller (issues #293–#297, +observability per #298). + +Everything below is written against the shipped manifests and the current code, +not against a generic kubebuilder layout. Two names differ from what you may +expect and both matter: + +- The controller's namespace on a hub is **`kubeslice-controller`**, not + `kubeslice-system`. `kubeslice-system` is a *worker* namespace (ADR #293 + Decision 1) and does not exist on a hub cluster at all. +- The Deployment is **`kubeslice-controller-manager`**. + +Substitute your own contexts for `` and `` throughout. + +--- + +## 0. Vocabulary: there are two Leases, and they are unrelated + +A hub cluster carries two `coordination.k8s.io/v1` Leases and confusing them +leads to the wrong conclusion every time: + +| Lease | What it is | +|---|---| +| `kubeslice-controller-ha` | **The HA lease.** Cross-cluster: which *hub* is Active. What a Standby watches and what promotion acquires. | +| `.kubeslice.io` (e.g. `6a2ced6b.kubeslice.io`) | controller-runtime's own `--leader-elect` lease — which *pod* leads inside one cluster. Nothing to do with HA. | + +``` +kubectl --context -n kubeslice-controller \ + get lease kubeslice-controller-ha \ + -o custom-columns=HOLDER:.spec.holderIdentity,RENEWED:.spec.renewTime +``` + +A stale `renewTime` on that Lease is the single fact the whole failover mechanism +turns on. + +--- + +## 1. Verify Active/Standby status + +### Reading the metrics endpoint + +The manager binds its metrics to `127.0.0.1:8080` and the `kube-rbac-proxy` +sidecar re-exposes them with TLS and authorization on `8443`. Two ways in. + +**Port-forward (simplest, and what to use while debugging).** `kubectl +port-forward` attaches to the pod's own network namespace, so the +loopback-bound port is reachable even though nothing outside the pod can dial +it directly: + +``` +kubectl --context -n kubeslice-controller \ + port-forward deploy/kubeslice-controller-manager 8080:8080 +``` + +``` +curl -s localhost:8080/metrics | grep kubeslice_controller_ha_ +``` + +**Through the proxied Service (what a scraper does).** Requires a token bound +to the `kubeslice-controller-metrics-reader` ClusterRole; the Service is +`kubeslice-controller-controller-manager-metrics-service:8443`. + +> If `/metrics` returns nothing through the sidecar, check that the manager is +> started with `--metrics-secure=false`. The sidecar is configured with +> `--upstream=http://127.0.0.1:8080/`, but that flag defaults to *true*, which +> makes controller-runtime serve TLS on 8080 — plain HTTP against a TLS listener, +> and the endpoint goes dark. `config/default/manager_auth_proxy_patch.yaml` +> sets it correctly; a Helm-deployed controller needs the same. + +### Which hub is Active + +``` +curl -s localhost:8080/metrics | grep kubeslice_controller_ha_leader_status +``` + +`1` = this instance holds leadership and its reconcilers are writing. `0` = it is +not writing. Run it against both hubs: **exactly one should report 1.** + +Two `1`s at once is split brain — go to §6. + +Two `0`s means no hub is reconciling. Either a promotion is in flight (check +`ha_promotion_duration_seconds`), or an Active has lost its Lease without a +Standby taking over (§5). + +### Is the Standby actually protecting anything + +A Standby that cannot read the Active's Lease will never promote, and looks +perfectly healthy until the day it is needed. Two gauges answer this: + +``` +curl -s localhost:8080/metrics | grep -E 'ha_armed|ha_remote_lease_age_seconds' +``` + +- `ha_armed 0` on a Standby → **it has never once read the Active's Lease and + cannot fail over.** Almost always credentials or RBAC; go to §5. +- `ha_remote_lease_age_seconds` → seconds since the Active last renewed. Should + hover below `--ha-lease-duration`. This is the leading indicator: it climbs + *before* a failover, so it is the one to alert on. + +### Recent transitions + +``` +kubectl --context -n kubeslice-controller get events \ + --field-selector reason=PromotedToActive +``` + +The full set of HA reasons, all recorded against the `kubeslice-controller-ha` +Lease so a single `get events -n kubeslice-controller` shows the lot: + +| Reason | Type | Meaning | +|---|---|---| +| `BecameActive` | Normal | Started in Active mode | +| `BecameStandby` | Normal | Started in Standby mode | +| `PromotedToActive` | Normal | Completed a promotion | +| `LeadershipLost` | Warning | Failed to renew past `--ha-renew-deadline`; stopped reconciling | +| `PromotionAborted` | Warning | Considered promoting and refused — see `ha_promotions_aborted_total` for which guard | +| `HAMirrorSyncFailed` | Warning | Mirror could not apply an object; retrying | + +> `HAMirrorSyncFailed` is the event issue #298's table calls `SyncError`. The +> name shipped earlier (#295) and is left alone rather than renamed under +> anyone's existing alerts. + +### How long has this hub been Active + +``` +curl -s localhost:8080/metrics \ + | grep kubeslice_controller_ha_last_promotion_timestamp_seconds +``` + +Subtract from `time()` for the age. Absent means *this process* has not promoted +— it does not mean the hub never did. Both this and `ha_failover_total` reset on +restart; the durable record is the `PromotedToActive` Event and the Lease's +`holderIdentity`. + +--- + +## 2. Simulate a failover + +A deliberate test. Expect a window of `--ha-lease-duration` + +`--ha-padding-seconds` in which no hub reconciles. + +**Before you start**, confirm the Standby is armed (§1) — otherwise you are +testing nothing and the "failure" will be permanent. + +``` +kubectl --context -n kubeslice-controller \ + logs deploy/kubeslice-controller-manager -c manager \ + --tail=20 | grep -i 'armed\|active hub lease' +``` + +Stop the Active: + +``` +kubectl --context -n kubeslice-controller \ + scale deploy/kubeslice-controller-manager --replicas=0 +``` + +Watch the Standby decide: + +``` +kubectl --context -n kubeslice-controller \ + logs -f deploy/kubeslice-controller-manager -c manager \ + | grep -iE 'stale|promot|leadership' +``` + +The sequence to expect, in this order: + +1. `active hub lease is STALE; evaluating promotion` +2. `state mirror stopped and confirmed exited` +3. `acquired lease on this hub` +4. `published activeController for the new Active` +5. `re-enqueued all reconciled types after promotion` +6. `PROMOTED to active` + +Then confirm on the promoted hub: + +``` +curl -s localhost:8080/metrics | grep -E 'ha_leader_status|ha_failover_total' +``` + +``` +kubectl --context -n kubeslice-controller \ + get lease kubeslice-controller-ha \ + -o jsonpath='{.spec.holderIdentity}' +``` + +And that workers were told: + +Cluster CRs live in the project namespace (`kubeslice-`, e.g. +`kubeslice-avesha`), not in `kubeslice-controller`: + +``` +kubectl --context -n \ + get cluster -o custom-columns=\ +NAME:.metadata.name,ACTIVE:.status.activeController.activeIdentity,\ +ENDPOINT:.status.activeController.endpoint +``` + +> Do **not** look for `status.conditions` on a Cluster CR. `ClusterStatus` has no +> `Conditions` field — only `clusterHealth.componentStatuses`, which is rebuilt +> from scratch on every pass. Worker-side connection health is tracked +> separately in worker-operator #469. + +### Restoring afterwards + +Bring the old Active back as a **Standby**, or you will have two Actives. It +needs `--ha-mode=standby` and a kubeconfig pointing at the newly promoted hub — +see §3, which is the same procedure. + +``` +kubectl --context -n kubeslice-controller \ + scale deploy/kubeslice-controller-manager --replicas=1 +``` + +> **Objects the demoted hub created itself do not go away, and cannot be deleted +> while it is a Standby.** They carry reconciler finalizers but no +> `ha.kubeslice.io/synced-from` label, so two rules combine against them: the +> mirror only manages objects it created, and the write fence stops this hub's +> reconcilers from clearing a finalizer. A `kubectl delete` against one hangs +> indefinitely with `deletionTimestamp` set and nothing to remove the finalizer. +> +> Both behaviours are correct — a Standby must not write, and the mirror must not +> delete objects it does not own — but the leftovers are real. To clear one: +> +> ``` +> kubectl -n patch \ +> --type=merge -p '{"metadata":{"finalizers":null}}' +> ``` +> +> Check for them after any role swap: +> +> ``` +> kubectl --context -n get \ +> -o json | jq -r '.items[] +> | select(.metadata.labels["ha.kubeslice.io/synced-from"] == null) +> | .metadata.name' +> ``` + +--- + +## 3. Rotate the Active kubeconfig credential + +The Standby reaches the Active with a kubeconfig in a Secret, mounted at the +path given by `--ha-active-kubeconfig`: + +| | | +|---|---| +| Secret | `ha-active-kubeconfig` in `kubeslice-controller` on the **Standby** | +| Key | `active.kubeconfig` | +| Mounted at | `/var/run/ha/active.kubeconfig` (per the deployment's flag) | + +Rotate when the credential expires, when the Active's API server certificate +changes, or **whenever the Active's address changes**. + +> **The trap, and the most common cause of a dead Standby.** This kubeconfig +> embeds both the Active's `server:` URL *and* its CA bundle. On Docker-based +> clusters (kind), node IPs are assigned in container start order and **are not +> stable across restarts** — a host reboot can permute them between clusters. +> The Standby then dials an address that now belongs to a *different* cluster, +> presenting a different CA, and fails with: +> +> ``` +> tls: failed to verify certificate: x509: certificate signed by unknown authority +> ``` +> +> That reads like a broken credential and is not one — it is a stale address. +> **Re-derive the IPs before regenerating anything:** +> +> ``` +> docker inspect -f \ +> '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ +> -control-plane +> ``` + +Check what the Standby currently believes: + +``` +kubectl --context -n kubeslice-controller \ + get secret ha-active-kubeconfig \ + -o jsonpath='{.data.active\.kubeconfig}' \ + | base64 -d | grep -E 'server:|certificate-authority' +``` + +Replace it: + +``` +kubectl --context -n kubeslice-controller \ + create secret generic ha-active-kubeconfig \ + --from-file=active.kubeconfig= \ + --dry-run=client -o yaml | kubectl --context apply -f - +``` + +The kubeconfig is read once at start-up, so the Standby must restart: + +``` +kubectl --context -n kubeslice-controller \ + rollout restart deploy/kubeslice-controller-manager +``` + +Nothing is lost by restarting. The mirror is rebuilt from the Active on every +start, and the reverse-diff prune pass reconciles anything missed while the +Standby was down. Confirm it came back armed: + +``` +curl -s localhost:8080/metrics | grep -E 'ha_armed|ha_remote_lease_reads_total' +``` + +`ha_remote_lease_reads_total{result="ok"}` must be climbing. If only +`result="error"` climbs, the new credential is not working either. + +The identity this kubeconfig authenticates as also needs read access **on the +Active hub** — see `config/ha/README.md`, which is applied there, not here. + +--- + +## 4. Troubleshoot: sync lag is high + +Symptom: the Standby's copy of the world is behind the Active's. + +**Distinguish "slow" from "stuck" first** — these have different causes and the +two metrics disagree deliberately: + +``` +curl -s localhost:8080/metrics \ + | grep -E 'ha_sync_lag_seconds|ha_sync_queue_depth|ha_sync_errors_total' +``` + +- **Lag high, depth low** → each object takes a long time; look at latency to the + Active hub. +- **Lag normal, depth climbing** → the syncer is keeping up with only a fraction + of the work. Lag is only observed for items that *completed*, so a healthy lag + figure here is survivorship bias. Raise `--ha-sync-workers`. +- **`ha_sync_errors_total` climbing** → real failures, retrying with backoff. + Break down by `kind` and `operation`; the matching `HAMirrorSyncFailed` Events + name the specific objects. + +Then check connectivity to the Active, which underlies all three: + +``` +curl -s localhost:8080/metrics \ + | grep -E 'ha_remote_lease_reads_total|ha_remote_lease_age_seconds' +``` + +Errors here mean the problem is the link or the credential (§3), not the syncer. + +### The drift backstop + +``` +curl -s localhost:8080/metrics | grep ha_prune_ +``` + +- `ha_prune_last_run_timestamp_seconds` far older than `--ha-sync-interval` → the + prune pass is not running. It waits for the remote cache to sync before its + first pass, so a cache that never synced leaves it silent. This is a + Standby-only series; its absence on an Active is correct, not a stall. +- `ha_prune_resurrected_total` climbing steadily → **zero is the healthy value.** + Prune re-enqueues objects the event path missed, so a backstop that fires + constantly means the informer path is dropping work. Worth investigating on its + own, not just tolerating. + +--- + +## 5. Troubleshoot: promotion is not firing + +The Active is gone and no Standby took over. Work down in order — the checks are +cheapest-first and each rules out the ones below it. + +**1. Is this hub even a Standby?** + +``` +kubectl --context -n kubeslice-controller \ + get deploy kubeslice-controller-manager \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].args}' +``` + +`--ha-mode=standby` must be present. Note that an *unrecognised* value is +rejected at start-up rather than silently treated as standalone, so a hub with a +typo will be crash-looping with `invalid HA configuration`, not running quietly. + +**2. Is it armed?** This is the single most common answer. + +``` +curl -s localhost:8080/metrics | grep ha_armed +``` + +`0` means it has never read the Active's Lease, and the arming rule then forbids +promotion by design — a hub that never saw the Active alive must never conclude +it died. Otherwise a mistyped namespace or a missing RBAC grant would become a +guaranteed split brain on first boot. Diagnose: + +``` +kubectl --context -n kubeslice-controller \ + logs deploy/kubeslice-controller-manager -c manager \ + | grep -i "never been read successfully" +``` + +That log line carries a hint listing the three things to check: +`--ha-active-kubeconfig`, RBAC for `coordination.k8s.io/leases` on the Active, +and the Lease namespace. Also see §3 for the stale-address trap. + +**3. Did it consider promoting and refuse?** + +``` +curl -s localhost:8080/metrics | grep ha_promotions_aborted_total +``` + +The `reason` label is the diagnosis: + +| Reason | What happened | What to do | +|---|---|---| +| `self_unhealthy` | This hub could not reach **its own** API server, so "the Active is gone" was equally consistent with *this* hub being broken | Fix this cluster; the refusal was correct | +| `lease_live` | The Active renewed between polls — the staleness verdict was a polling race | Nothing. The Active is alive | +| `mirror_not_stopped` | The mirror did not confirm it stopped within `--ha-promotion-grace-period` | See the warning below | +| `lease_acquire_failed` | Could not write the Lease on its own cluster | Fix this cluster's API server; it retries next tick | +| `already_promoting` | A concurrent tick held the latch | Nothing; benign | +| `no_remote_client` | Asked to promote with no client to the Active | Configuration; should be unreachable from the watch loop | + +Each of these also emits a `PromotionAborted` Warning Event. + +> **`mirror_not_stopped` needs a follow-up.** An abort at that point leaves a +> Standby that has stopped mirroring and cannot restart it — promotion holds only +> a one-way stop handle. Later attempts still work, but if the Active recovers +> first, the guards will (correctly) refuse to promote and this hub stays a +> Standby whose mirror is dead, drifting further from the Active. The log says so +> at error level: *"promotion aborted after the state mirror was stopped"*. +> **Restart the Standby** to resume mirroring. + +**4. Is detection just slower than you expected?** + +``` +curl -s localhost:8080/metrics | grep ha_failover_detection_seconds +``` + +Detection cannot beat `--ha-lease-duration` + `--ha-padding-seconds`, plus up to +one `--ha-retry-period` of polling granularity. If that budget is too slow for +you, those are the flags to shorten — at the cost of promoting on shorter +evidence. + +**5. Did promotion start and stall?** + +``` +curl -s localhost:8080/metrics | grep ha_promotion_step_duration_seconds +``` + +The `step` label localises it: `stop_mirror`, `acquire_lease`, +`publish_active_controller`, `kick_reconcilers`, `emit_event`. A step sitting at +the `--ha-promotion-grace-period` ceiling is the one waiting on something. + +--- + +## 6. Split brain: both hubs report leadership + +`ha_leader_status 1` on both hubs. Objects overwrite each other, prune's reverse +diff resurrects deletions, and workers receive contradictory instructions. + +**Split brain is an explicit non-goal of ADR #293 Decision 8** — there is no +fencing token or quorum here. A sustained network partition between hubs *can* +produce it, and recovery is manual. + +Pick the hub to keep — normally the one with the newer +`ha_last_promotion_timestamp_seconds`, or whichever workers are actually +reporting to. Then: + +1. Scale the loser to 0. +2. Confirm the survivor holds `kubeslice-controller-ha` and reports + `ha_leader_status 1`. +3. Check `status.activeController.activeIdentity` on every Cluster CR names the + survivor. +4. Reconcile divergence by hand — objects written on the loser during the + partition are not merged by anything. +5. Bring the loser back as a Standby (§3). + +--- + +## Alerting starting points + +| Condition | Meaning | +|---|---| +| `sum(kubeslice_controller_ha_leader_status) != 1` | No Active, or two | +| `kubeslice_controller_ha_armed == 0` on a Standby | HA is not actually protecting anything | +| `kubeslice_controller_ha_remote_lease_age_seconds` > ½ the failover budget | Leading indicator; fires before a failover | +| `time() - kubeslice_controller_ha_lease_last_renew_time_seconds` > `--ha-renew-deadline` | Active is about to drop leadership | +| `rate(kubeslice_controller_ha_remote_lease_reads_total{result="error"}[5m]) > 0` | Standby is losing sight of the Active | +| `increase(kubeslice_controller_ha_promotions_aborted_total[1h]) > 0` | A takeover was considered and refused | +| `increase(kubeslice_controller_ha_prune_resurrected_total[1h]) > 0` | The event path is dropping work | +| `increase(kubeslice_controller_ha_active_publish_errors_total[15m]) > 0` | Failover may work without any worker noticing | + +Do not alert on the value of a timestamp gauge — alert on its age. + +**Role-scoped metrics carry a `mode` label and exist only on the role they +describe**, which is what makes the expressions above safe to write without +filtering by hub: + +| Metric | Published by | +|---|---| +| `ha_lease_last_renew_time_seconds{mode="active"}` | an Active only | +| `ha_armed{mode="standby"}` | a Standby only | +| `ha_remote_lease_age_seconds{mode="standby"}` | a Standby only (dropped on promotion) | +| `ha_prune_last_run_timestamp_seconds{mode="standby"}` | a Standby only | +| `ha_last_promotion_timestamp_seconds{mode="active"}` | only after this process has promoted | + +So `ha_armed == 0` matches Standbys and nothing else, and an *absent* +`ha_prune_last_run_timestamp_seconds` on an Active is correct rather than a +stalled backstop. `ha_leader_status` is the deliberate exception — both roles +publish it, because `sum(...) != 1` has to be expressible across the pair. + +The reason absence is engineered rather than assumed: a plain registered gauge +always reports `0`, so simply not setting one is not the same as not having it. A +zeroed timestamp reads as 1970, and `time() - metric` then returns decades and +fires forever. Each of these is a labelled vector so that the series genuinely +does not exist on the wrong role. diff --git a/events/events_generated.go b/events/events_generated.go index 264c43b7e..4b2af08f1 100644 --- a/events/events_generated.go +++ b/events/events_generated.go @@ -734,6 +734,54 @@ var EventsMap = map[events.EventName]*events.EventSchema{ ReportingController: "controller", Message: "Warning - Certificate Creation job Failed", }, + "HAMirrorSyncFailed": { + Name: "HAMirrorSyncFailed", + Reason: "HAMirrorSyncFailed", + Action: "HAMirrorSync", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "Failed to mirror a resource from the Active hub onto the Standby; the syncer will retry.", + }, + "HAPromotedToActive": { + Name: "HAPromotedToActive", + Reason: "PromotedToActive", + Action: "HAPromotion", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "This hub was promoted from Standby to Active after the previous Active hub's lease went stale.", + }, + "HABecameActive": { + Name: "HABecameActive", + Reason: "BecameActive", + Action: "HAStartup", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "This hub started in Active mode and will hold the HA lease and reconcile.", + }, + "HABecameStandby": { + Name: "HABecameStandby", + Reason: "BecameStandby", + Action: "HAStartup", + Type: events.EventTypeNormal, + ReportingController: "controller", + Message: "This hub started in Standby mode; it mirrors the Active hub's state and does not reconcile.", + }, + "HALeadershipLost": { + Name: "HALeadershipLost", + Reason: "LeadershipLost", + Action: "HALeaseRenewal", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "This hub failed to renew its HA lease within the renew deadline and has released leadership; it will not reconcile until it renews again.", + }, + "HAPromotionAborted": { + Name: "HAPromotionAborted", + Reason: "PromotionAborted", + Action: "HAPromotion", + Type: events.EventTypeWarning, + ReportingController: "controller", + Message: "A promotion was considered and refused; see ha_promotions_aborted_total and the controller logs for which guard fired.", + }, } var ( @@ -826,4 +874,10 @@ var ( EventCertificatesRenewNow events.EventName = "CertificatesRenewNow" EventIllegalVPNKeyRotationConfigDelete events.EventName = "IllegalVPNKeyRotationConfigDelete" EventCertificateJobFailed events.EventName = "CertificateJobFailed" + EventHAMirrorSyncFailed events.EventName = "HAMirrorSyncFailed" + EventHAPromotedToActive events.EventName = "HAPromotedToActive" + EventHABecameActive events.EventName = "HABecameActive" + EventHABecameStandby events.EventName = "HABecameStandby" + EventHALeadershipLost events.EventName = "HALeadershipLost" + EventHAPromotionAborted events.EventName = "HAPromotionAborted" ) diff --git a/main.go b/main.go index 5d9043b14..74002d389 100644 --- a/main.go +++ b/main.go @@ -17,11 +17,14 @@ package main import ( + "context" "crypto/tls" "flag" "fmt" "os" "path/filepath" + "time" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" "sigs.k8s.io/controller-runtime/pkg/webhook" @@ -42,9 +45,13 @@ import ( "github.com/kubeslice/kubeslice-controller/controllers/controller" "github.com/kubeslice/kubeslice-controller/controllers/worker" "github.com/kubeslice/kubeslice-controller/metrics" + "github.com/kubeslice/kubeslice-controller/pkg/ha" "github.com/kubeslice/kubeslice-controller/service" "github.com/kubeslice/kubeslice-controller/util" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" //+kubebuilder:scaffold:imports ) @@ -110,6 +117,20 @@ func initialize(services *service.Services) { var jobServiceAccount string // get prometheus endpoint from environment var prometheusServiceEndpoint string + // HA (Active/Standby cross-cluster) configuration — see ADR #293 / issue #294 + var haMode string + var haIdentity string + var haActiveKubeconfig string + var haLeaseNamespace string + var haLeaseDuration time.Duration + var haRenewDeadline time.Duration + var haRetryPeriod time.Duration + var haPaddingSeconds time.Duration + var haSyncWorkers int + var haSyncInterval time.Duration + var haSelfCABundlePath string + var haPromotionDialTimeout time.Duration + var haPromotionGracePeriod time.Duration flag.StringVar(&rbacResourcePrefix, "rbac-resource-prefix", service.RbacResourcePrefix, "RBAC resource prefix") flag.StringVar(&projectNameSpacePrefixFromCustomer, "project-namespace-prefix", service.ProjectNamespacePrefix, fmt.Sprintf("Overrides the default %s kubeslice namespace", service.ProjectNamespacePrefix)) @@ -138,6 +159,21 @@ func initialize(services *service.Services) { "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + // Cross-cluster HA flags. --ha-mode=standalone (default) preserves today's behaviour. + flag.StringVar(&haMode, "ha-mode", "standalone", `Cross-cluster HA mode: "active", "standby", or "standalone" (default).`) + flag.StringVar(&haIdentity, "ha-identity", "", "Stable per-cluster identity recorded in the Lease (defaults to the hostname).") + flag.StringVar(&haActiveKubeconfig, "ha-active-kubeconfig", "", "Path to the Active hub kubeconfig; required in standby mode.") + flag.StringVar(&haLeaseNamespace, "ha-lease-namespace", os.Getenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE"), "Namespace for the HA Lease; defaults to the controller's own namespace (KUBESLICE_CONTROLLER_MANAGER_NAMESPACE), where the leader-election Role grants leases. Empty falls back to the pkg/ha default.") + flag.DurationVar(&haLeaseDuration, "ha-lease-duration", ha.DefaultLeaseDuration, "HA Lease duration.") + flag.DurationVar(&haRenewDeadline, "ha-renew-deadline", ha.DefaultRenewDeadline, "Deadline for the Active to renew its Lease before releasing leadership.") + flag.DurationVar(&haRetryPeriod, "ha-retry-period", ha.DefaultRetryPeriod, "Interval between Lease renew/watch attempts.") + flag.DurationVar(&haPaddingSeconds, "ha-padding-seconds", ha.DefaultPaddingSeconds, "Extra buffer a Standby waits before treating the Active Lease as stale.") + flag.IntVar(&haSyncWorkers, "ha-sync-workers", ha.DefaultSyncWorkers, "Number of workers draining the Standby's remote-mirror workqueue.") + flag.DurationVar(&haSyncInterval, "ha-sync-interval", ha.DefaultPruneInterval, "How often the Standby prunes mirrored objects that no longer exist on the Active hub.") + flag.DurationVar(&haPromotionDialTimeout, "ha-promotion-dial-timeout", ha.DefaultPromotionDialTimeout, "Bound on every read a Standby makes of a Lease over the network: each periodic poll of the Active's Lease, its own self-health check, and the final dial. Unbounded, an API server that accepts the connection and then stops answering blocks the watch loop and stalls detection entirely.") + flag.DurationVar(&haPromotionGracePeriod, "ha-promotion-grace-period", ha.DefaultPromotionGracePeriod, "Bound on each step of the promotion sequence that waits on another component: stopping the mirror, publishing status.activeController, re-enqueuing objects, and emitting the event. A sequencing budget, unrelated to --ha-padding-seconds.") + flag.StringVar(&haSelfCABundlePath, "ha-self-ca-bundle-path", ha.DefaultSelfCABundlePath, "Path to this hub's own API server CA, published in status.activeController.caBundle. Unreadable is not fatal; publication continues without it.") + flag.Parse() // initialize logger @@ -278,8 +314,84 @@ func initialize(services *service.Services) { }) // setting up metrics collector go metrics.StartMetricsCollector(service.MetricPort, true) + + // Set up cross-cluster HA leader election (ADR #293 / issue #294). In + // standalone mode (the default) the elector is always the leader, so the + // reconciler write-fence is a no-op and behaviour is unchanged. + // Rejected rather than coerced: a mistyped mode that silently became + // standalone would start a second unconditionally-unfenced writer against + // the same worker clusters as the real Active. + haRunMode, err := ha.ParseHAModeStrict(haMode) + if err != nil { + setupLog.Error(err, "invalid HA configuration") + os.Exit(1) + } + localHAClient, err := client.New(mgr.GetConfig(), client.Options{Scheme: scheme}) + if err != nil { + setupLog.Error(err, "unable to build HA local client") + os.Exit(1) + } + var remoteHAClient client.Client + var remoteHACfg *rest.Config + if haRunMode == ha.ModeStandby { + if haActiveKubeconfig == "" { + setupLog.Error(fmt.Errorf("missing --ha-active-kubeconfig"), "standby mode requires the Active hub kubeconfig") + os.Exit(1) + } + var cfgErr error + remoteHACfg, cfgErr = clientcmd.BuildConfigFromFlags("", haActiveKubeconfig) + if cfgErr != nil { + setupLog.Error(cfgErr, "unable to load Active hub kubeconfig", "path", haActiveKubeconfig) + os.Exit(1) + } + remoteHAClient, cfgErr = client.New(remoteHACfg, client.Options{Scheme: scheme}) + if cfgErr != nil { + setupLog.Error(cfgErr, "unable to build remote client for Active hub") + os.Exit(1) + } + } + leaderElector := ha.NewClusterLeaderElector(localHAClient, remoteHAClient, ha.Options{ + Mode: haRunMode, + Identity: haIdentity, + LeaseNamespace: haLeaseNamespace, + LeaseDuration: haLeaseDuration, + RenewDeadline: haRenewDeadline, + RetryPeriod: haRetryPeriod, + PaddingSeconds: haPaddingSeconds, + PromotionDialTimeout: haPromotionDialTimeout, + PromotionGracePeriod: haPromotionGracePeriod, + EventRecorder: eventRecorder, + Log: controllerLog.With("name", "ha"), + }) + setupLog.Info("high availability configured", "mode", haRunMode, "identity", leaderElector.Identity()) + + // RemoteSyncer mirrors CRDMirrorSet from the Active hub onto this + // cluster; a no-op in any mode other than standby (issue #295). Reuses + // the same remote config and local client the elector above already + // built rather than loading the kubeconfig twice. + remoteSyncer, err := ha.NewRemoteSyncer(localHAClient, remoteHACfg, scheme, haRunMode, ha.RemoteSyncerOptions{ + Resources: ha.FullMirrorSet(), + Workers: haSyncWorkers, + PruneInterval: haSyncInterval, + EventRecorder: eventRecorder, + Log: controllerLog.With("name", "ha-remote-syncer"), + }) + if err != nil { + setupLog.Error(err, "unable to build HA remote syncer") + os.Exit(1) + } + + // One channel per reconciled type, delivered to each controller below and + // filled once on promotion. Built unconditionally: outside HA the kick + // simply never fires, and wiring it here keeps the reconcilers identical in + // both modes. + reconcileKicker := ha.NewReconcileKicker(mgr.GetClient(), ha.ReconciledGVKs(), + controllerLog.With("name", "ha-reconcile-kicker")) + // initialize controller with Project Kind if err = (&controller.ProjectReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKProject), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "Project"), @@ -291,6 +403,8 @@ func initialize(services *service.Services) { } // initialize controller with Cluster Kind if err = (&controller.ClusterReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKCluster), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "Cluster"), @@ -302,6 +416,8 @@ func initialize(services *service.Services) { } // initialize controller with SliceConfig Kind if err = (&controller.SliceConfigReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKSliceConfig), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "SliceConfig"), @@ -313,6 +429,8 @@ func initialize(services *service.Services) { } // initialize controller with ServiceExportConfig Kind if err = (&controller.ServiceExportConfigReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKServiceExportConfig), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "ServiceExportConfig"), @@ -323,6 +441,8 @@ func initialize(services *service.Services) { os.Exit(1) } if err = (&worker.WorkerSliceGatewayReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKWorkerSliceGateway), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "WorkerSliceGateway"), @@ -333,6 +453,8 @@ func initialize(services *service.Services) { os.Exit(1) } if err = (&worker.WorkerSliceConfigReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKWorkerSliceConfig), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "WorkerSliceConfig"), @@ -343,6 +465,8 @@ func initialize(services *service.Services) { os.Exit(1) } if err = (&worker.WorkerServiceImportReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKWorkerServiceImport), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "WorkerServiceImport"), @@ -353,6 +477,8 @@ func initialize(services *service.Services) { os.Exit(1) } if err = (&controller.SliceQoSConfigReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKSliceQoSConfig), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "SliceQoSConfig"), @@ -363,6 +489,8 @@ func initialize(services *service.Services) { os.Exit(1) } if err = (&controller.VpnKeyRotationReconciler{ + PromotionKick: reconcileKicker.Source(ha.GVKVpnKeyRotation), + LeaderElector: leaderElector, Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Log: controllerLog.With("name", "VpnKeyRotationConfig"), @@ -419,8 +547,95 @@ func initialize(services *service.Services) { os.Exit(1) } + ctx := ctrl.SetupSignalHandler() + + // Publish status.activeController for as long as this hub holds leadership + // (ADR #293 Decision 7), so workers can identify the Active by watching both + // hubs. Deliberately not started in standalone mode: a non-HA deployment must + // leave the field absent, which is what keeps an existing worker's behaviour + // unchanged. A Standby starts it too — the publisher no-ops while it is not + // the leader, so promotion needs no extra wiring here. + var activePublisher *ha.ActivePublisher + if haRunMode != ha.ModeStandalone { + activePublisher = ha.NewActivePublisher(localHAClient, leaderElector, ha.ActivePublisherOptions{ + Endpoint: controllerEndpoint, + CABundlePath: haSelfCABundlePath, + Log: controllerLog.With("name", "ha-active-publisher"), + }) + go func() { + if err := activePublisher.Start(ctx); err != nil { + setupLog.Error(err, "HA activeController publisher exited") + } + }() + } + + // BecameActive / BecameStandby (issue #298). Emitted here rather than beside + // the elector's construction for two reasons: ctx does not exist until + // SetupSignalHandler above, and recording an Event is an API-server write that + // building an elector should not perform — every unit test constructs one. A + // no-op in standalone mode and whenever no recorder is configured. + leaderElector.EmitStartupModeEvent(ctx) + + // Start the HA background loop for the configured mode. Standalone starts + // nothing (it is always the leader). + switch haRunMode { + case ha.ModeActive: + go func() { + if err := leaderElector.StartLeaseRenewal(ctx); err != nil { + setupLog.Error(err, "HA lease renewal loop exited") + } + }() + case ha.ModeStandby: + // The syncer gets its own cancellable context so promotion can stop the + // mirror without tearing down everything else that hangs off ctx. + syncerCtx, stopSyncer := context.WithCancel(ctx) + syncerDone := make(chan struct{}) + go func() { + defer close(syncerDone) + if err := remoteSyncer.Start(syncerCtx); err != nil { + setupLog.Error(err, "HA remote syncer exited") + } + }() + + // Promotion's effects outside the elector (issue #297). Wired here rather + // than at construction because the syncer, the publisher and the manager + // are all built after the elector. + leaderElector.SetPromotionHooks(ha.PromotionHooks{ + // Cancel the mirror and WAIT for it to confirm it has stopped. The + // wait is the point: RemoteSyncer.Start already drains its workqueue + // and prune goroutine before returning, so returning from it is a + // sufficient and already-correct barrier. Without the wait, a hub can + // open its write fence while the mirror is still writing — and in the + // most common failover trigger (the Active's pod dies, its API server + // does not) the mirror is very much still alive at that moment. + StopMirror: func(promoteCtx context.Context) error { + stopSyncer() + select { + case <-syncerDone: + return nil + case <-promoteCtx.Done(): + return fmt.Errorf("timed out waiting for the state mirror to stop: %w", promoteCtx.Err()) + } + }, + KickReconcilers: reconcileKicker.Kick, + PublishActiveController: func(promoteCtx context.Context) error { + if activePublisher == nil { + return nil + } + return activePublisher.PublishOnce(promoteCtx) + }, + EmitPromotedEvent: ha.PromotedToActiveEmitter(eventRecorder), + }) + + go func() { + if err := leaderElector.WatchRemoteLease(ctx); err != nil { + setupLog.Error(err, "HA remote lease watch loop exited") + } + }() + } + setupLog.Info("starting manager") - if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err = mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } @@ -438,8 +653,13 @@ func initialize(services *service.Services) { //+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;create;update;patch;delete;escalate +//+kubebuilder:rbac:groups="",resources=namespaces/status,verbs=get;update;patch //+kubebuilder:rbac:groups="",resources=secrets,verbs=create;get;list;watch;escalate;update;patch;delete //+kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;escalate;update;patch;create //+kubebuilder:rbac:groups="batch",resources=jobs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings;roles;clusterroles,verbs=get;list;watch;create;update;patch;delete + +// NOTE: the HA leader-election Lease lives in the controller's own namespace and +// reuses the existing leader-election Role's coordination.k8s.io/leases grant +// (config/rbac/leader_election_role.yaml); no dedicated RBAC marker is needed. diff --git a/pkg/ha/active_publisher.go b/pkg/ha/active_publisher.go new file mode 100644 index 000000000..e188c4660 --- /dev/null +++ b/pkg/ha/active_publisher.go @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "os" + "time" + + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/util" +) + +const ( + // DefaultActivePublishInterval is how often the leader re-checks that every + // Cluster CR carries its declaration. Convergence after a promotion does not + // wait for this tick — promotion runs one synchronous PublishOnce. + DefaultActivePublishInterval = 30 * time.Second + + // DefaultLeadershipPollInterval is how often a hub that is not (yet) the + // leader re-checks whether it has become one. It is deliberately much shorter + // than the publish interval: an Active acquires its Lease a second or two + // after start-up, and waiting a full publish interval to notice would leave a + // freshly started hub unadvertised for that whole window. Costs nothing while + // idle — a non-leader returns before touching the API server. + DefaultLeadershipPollInterval = 2 * time.Second + + // DefaultSelfCABundlePath is where a pod finds its own API server's CA. + DefaultSelfCABundlePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + + // PlaceholderControllerEndpoint is the value service.ControllerEndpoint ships + // with when --controller-end-point is not set. It is a documentation + // placeholder that resolves to nothing, so publishing it as the failover + // target would be worse than publishing nothing at all. + // + // It is duplicated here rather than imported because main.go overwrites + // service.ControllerEndpoint with the flag value at startup, leaving no way + // to recover the default afterwards. TestPlaceholderMatchesServiceDefault + // pins the two together. + PlaceholderControllerEndpoint = "https://controller.cisco.com:6443/" +) + +// leadership is the subset of ClusterLeaderElector the publisher depends on. +// Narrowing it keeps the publisher testable without a live Lease. +type leadership interface { + IsLeader() bool + Identity() string +} + +// ActivePublisherOptions configures an ActivePublisher. Zero-valued fields fall +// back to the Default* constants. +type ActivePublisherOptions struct { + // Endpoint is this hub's own API server endpoint, as handed to workers at + // registration (service.ControllerEndpoint / --controller-end-point). Reusing + // that value keeps one source of truth for "where this hub is reachable". + Endpoint string + // CABundlePath is this hub's own API server CA, read once at startup. + CABundlePath string + Interval time.Duration + // LeadershipPollInterval is how often to re-check for leadership while this + // hub does not hold it. Defaults to DefaultLeadershipPollInterval. + LeadershipPollInterval time.Duration + Log *zap.SugaredLogger +} + +// ActivePublisher keeps status.activeController current on every Cluster CR on +// this hub, for as long as this hub holds leadership (ADR #293 Decision 7). +// +// It is a standalone loop rather than a step inside ClusterService.ReconcileCluster +// because it has to converge independently of reconciler traffic — and reconciler +// traffic is exactly what is absent immediately after a promotion, when the write +// fence has just opened but nothing has re-enqueued the pre-existing objects yet. +type ActivePublisher struct { + localClient client.Client + elector leadership + + endpoint string + caBundle string + interval time.Duration + leaderPoll time.Duration + + log *zap.SugaredLogger +} + +// NewActivePublisher builds a publisher writing to local, gated on elector. +// +// A missing or unreadable CA bundle is not fatal: the endpoint and identity are +// what a worker needs to select a hub, and a worker that already pins the hub's +// CA does not need it republished. The failure is logged and publication +// continues without it. +func NewActivePublisher(local client.Client, elector leadership, opts ActivePublisherOptions) *ActivePublisher { + if opts.Interval == 0 { + opts.Interval = DefaultActivePublishInterval + } + if opts.LeadershipPollInterval == 0 { + opts.LeadershipPollInterval = DefaultLeadershipPollInterval + } + if opts.CABundlePath == "" { + opts.CABundlePath = DefaultSelfCABundlePath + } + if opts.Log == nil { + opts.Log = util.NewLogger().With("name", "ha-active-publisher") + } + + p := &ActivePublisher{ + localClient: local, + elector: elector, + endpoint: opts.Endpoint, + interval: opts.Interval, + leaderPoll: opts.LeadershipPollInterval, + log: opts.Log, + } + + if ca, err := os.ReadFile(opts.CABundlePath); err != nil { + p.log.Warnw("could not read own CA bundle; publishing activeController without it", + "path", opts.CABundlePath, "error", err) + } else { + p.caBundle = base64.StdEncoding.EncodeToString(ca) + } + return p +} + +// Start publishes immediately, then every interval, until ctx is cancelled. +// Returns nil on cancellation so a graceful shutdown is not logged as an error. +func (p *ActivePublisher) Start(ctx context.Context) error { + p.log.Infow("starting activeController publisher", + "endpoint", p.endpoint, "interval", p.interval, "haveCABundle", p.caBundle != "") + + for { + // The wait is chosen from what this pass actually did, not from a second + // IsLeader() read: leadership arriving between the two would otherwise + // still cost a full publish interval. + // + // While this hub is not the leader there is nothing to publish, but + // leadership can arrive at any moment — on an Active, a second or two + // after start-up; on a Standby, at promotion. Re-checking on the short + // interval is what makes a freshly started or freshly promoted hub + // advertise itself promptly. Verified live: on the publish interval + // alone a fresh Active took 31s to appear. + wasLeader, err := p.publishOnce(ctx) + if err != nil { + // Counted here rather than inside publish(), which promotion also + // calls — promotion increments it on its own failure path so that one + // failed publication is one increment regardless of which caller made + // it. Counting inside publish() would double-count promotion's. + haActivePublishErrorsTotal.Inc() + p.log.Warnw("activeController publication failed; will retry", "error", err) + } + wait := p.interval + if !wasLeader { + wait = p.leaderPoll + } + timer := time.NewTimer(wait) + select { + case <-ctx.Done(): + timer.Stop() + p.log.Infow("activeController publisher stopped", "reason", ctx.Err()) + return nil + case <-timer.C: + } + } +} + +// PublishOnce runs a single pass over every Cluster CR on this hub. Promotion +// calls it synchronously, at the point where it has already taken the Lease and +// switched to Active but has not yet opened the write fence, so a failover does +// not wait for the publisher's next tick. +// +// It deliberately does NOT gate on IsLeader(). That reads like a safety check +// and is in fact the opposite: promotion holds the write-fence latch across its +// whole sequence, so IsLeader() reports false for exactly the window in which +// this is called. Gating here made the promotion-time publication a silent +// no-op that still logged success, leaving status.activeController naming the +// dead hub until the periodic loop next ran. +// +// The caller is responsible for having established leadership first. The +// periodic loop must keep using publishOnce, which does gate — an Active that +// has lost its Lease must stop advertising itself. +func (p *ActivePublisher) PublishOnce(ctx context.Context) error { + return p.publish(ctx) +} + +// publishOnce is the leadership-gated pass the periodic loop runs. It also +// reports whether this hub held leadership, which Start uses to decide how long +// to wait next. +// +// A Standby must never write this field: its copy is owned by the state mirror +// and names the Active, which is the whole rule workers use to tell the two +// hubs apart. +func (p *ActivePublisher) publishOnce(ctx context.Context) (leader bool, err error) { + if !p.elector.IsLeader() { + p.log.Debugw("not the leader; skipping activeController publication") + return false, nil + } + return true, p.publish(ctx) +} + +// publish performs one ungated pass over every Cluster CR on this hub. +func (p *ActivePublisher) publish(ctx context.Context) error { + if err := p.validEndpoint(); err != nil { + // Deliberately not fatal. A hub that cannot describe itself should keep + // reconciling; it just must not advertise an address nobody can reach. + p.log.Errorw("refusing to publish activeController", "endpoint", p.endpoint, "error", err) + return nil + } + + clusters := &controllerv1alpha1.ClusterList{} + if err := p.localClient.List(ctx, clusters); err != nil { + return fmt.Errorf("listing clusters to publish activeController: %w", err) + } + + desired := controllerv1alpha1.ActiveControllerInfo{ + Endpoint: p.endpoint, + CABundle: p.caBundle, + ActiveIdentity: p.elector.Identity(), + } + + var errs []error + updated := 0 + for i := range clusters.Items { + cluster := &clusters.Items[i] + if activeControllerUpToDate(cluster.Status.ActiveController, desired) { + continue + } + payload := desired + payload.LastUpdated = metav1.Now() + cluster.Status.ActiveController = &payload + if err := p.localClient.Status().Update(ctx, cluster); err != nil { + errs = append(errs, fmt.Errorf("cluster %s/%s: %w", cluster.Namespace, cluster.Name, err)) + continue + } + updated++ + } + if updated > 0 { + p.log.Infow("published activeController", + "clusters", updated, "identity", desired.ActiveIdentity, "endpoint", desired.Endpoint) + } + return errors.Join(errs...) +} + +// validEndpoint rejects the two values that must never reach a worker: nothing, +// and the shipped placeholder. +func (p *ActivePublisher) validEndpoint() error { + switch p.endpoint { + case "": + return errors.New("controller endpoint is empty; set --controller-end-point on an HA deployment") + case PlaceholderControllerEndpoint: + return errors.New("controller endpoint is still the shipped placeholder; set --controller-end-point on an HA deployment") + } + return nil +} + +// activeControllerUpToDate compares everything except LastUpdated. Including the +// timestamp would make every pass differ from itself, turning a convergence check +// into a write to every Cluster CR on every tick. +// +// Note what is deliberately absent: nothing ever clears this field. A hub only +// stops publishing by losing leadership, which in practice means it stopped +// renewing its Lease — it is unreachable, so a worker cannot read the stale +// declaration anyway. Auto-demotion of a recovered hub is an explicit ADR #293 +// non-goal (Decision 8); LastUpdated is what lets a consumer prefer the fresher +// of two claims if it ever does see both. +func activeControllerUpToDate(current *controllerv1alpha1.ActiveControllerInfo, desired controllerv1alpha1.ActiveControllerInfo) bool { + return current != nil && + current.Endpoint == desired.Endpoint && + current.CABundle == desired.CABundle && + current.ActiveIdentity == desired.ActiveIdentity +} diff --git a/pkg/ha/active_publisher_test.go b/pkg/ha/active_publisher_test.go new file mode 100644 index 000000000..e017966a2 --- /dev/null +++ b/pkg/ha/active_publisher_test.go @@ -0,0 +1,376 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1" + "github.com/kubeslice/kubeslice-controller/service" +) + +// stubLeadership stands in for ClusterLeaderElector so the publisher can be +// exercised without a live Lease. +type stubLeadership struct { + leader bool + identity string +} + +func (s stubLeadership) IsLeader() bool { return s.leader } +func (s stubLeadership) Identity() string { return s.identity } + +// clusterScheme extends the shared testScheme with the controller CRDs the +// publisher writes to. +func clusterScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, controllerv1alpha1.AddToScheme(scheme)) + return scheme +} + +func newCluster(name, namespace string) *controllerv1alpha1.Cluster { + return &controllerv1alpha1.Cluster{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + } +} + +// clusterClient builds a fake client that honours the status subresource, which +// the publisher writes through exclusively. +func clusterClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder(). + WithScheme(clusterScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&controllerv1alpha1.Cluster{}). + Build() +} + +// countingClusterClient wraps clusterClient and counts status writes, so a test +// can assert that a converged pass writes nothing at all. +func countingClusterClient(t *testing.T, writes *int, objs ...client.Object) client.Client { + t.Helper() + base := fake.NewClientBuilder(). + WithScheme(clusterScheme(t)). + WithObjects(objs...). + WithStatusSubresource(&controllerv1alpha1.Cluster{}). + Build() + return interceptor.NewClient(base, interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + *writes++ + return c.Status().Update(ctx, obj) + }, + }) +} + +func testPublisher(t *testing.T, c client.Client, elector leadership, endpoint string) *ActivePublisher { + t.Helper() + return NewActivePublisher(c, elector, ActivePublisherOptions{ + Endpoint: endpoint, + // A path that cannot exist, so the CA-bundle read fails predictably and + // the tests that care about it opt in explicitly. + CABundlePath: filepath.Join(t.TempDir(), "absent-ca.crt"), + Log: testLog(), + }) +} + +func getCluster(t *testing.T, c client.Client, name, namespace string) *controllerv1alpha1.Cluster { + t.Helper() + got := &controllerv1alpha1.Cluster{} + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Name: name, Namespace: namespace}, got)) + return got +} + +// TestPublishOnce_WritesWhileTheWriteFenceIsShut is the regression test for a +// bug live testing caught: promotion calls PublishOnce at step 7, between +// taking the Lease and opening the write fence, and IsLeader() reports false +// for that entire window because promote() holds the fence latch across its +// whole sequence. Gating PublishOnce on IsLeader() therefore made step 7 a +// guaranteed no-op that still logged success, and status.activeController kept +// naming the dead hub until the periodic loop happened to run. +// +// The gate belongs on the periodic loop (see publishOnce), not here. +func TestPublishOnce_WritesWhileTheWriteFenceIsShut(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: false, identity: "hub-b"}, "https://hub-b.example.com:6443") + + require.NoError(t, p.PublishOnce(context.Background())) + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + require.NotNil(t, got.Status.ActiveController, + "promotion must be able to publish before it opens the fence") + assert.Equal(t, "hub-b", got.Status.ActiveController.ActiveIdentity) + assert.Equal(t, "https://hub-b.example.com:6443", got.Status.ActiveController.Endpoint) +} + +// TestPublishOnce_PeriodicLoopStillSkipsWhenNotLeader is the other half, and +// carries the invariant that matters: a Standby must never write its own +// identity here. The mirror owns a Standby's copy of the field, and a worker +// tells the two hubs apart by which one names itself — so the loop that runs +// continuously in standby mode has to stay gated, and a hub that has lost its +// Lease has to stop advertising itself. +// +// Ungating PublishOnce does not weaken that: the only caller is the promotion +// sequence, which reaches it after taking the Lease and setting mode Active, at +// which point the hub is not a Standby any more. +func TestPublishOnce_PeriodicLoopStillSkipsWhenNotLeader(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: false, identity: "hub-b"}, "https://hub-b.example.com:6443") + + leader, err := p.publishOnce(context.Background()) + require.NoError(t, err) + assert.False(t, leader, "the loop must report it did not hold leadership") + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + assert.Nil(t, got.Status.ActiveController, + "a hub that does not hold leadership must not advertise itself from the periodic loop") +} + +func TestPublishOnce_WritesActiveControllerToEveryCluster(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha"), newCluster("worker-2", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, "https://hub-a.example.com:6443") + + require.NoError(t, p.PublishOnce(context.Background())) + + for _, name := range []string{"worker-1", "worker-2"} { + got := getCluster(t, c, name, "kubeslice-avesha") + require.NotNil(t, got.Status.ActiveController, "publisher must declare on %s", name) + assert.Equal(t, "https://hub-a.example.com:6443", got.Status.ActiveController.Endpoint) + assert.Equal(t, "hub-a", got.Status.ActiveController.ActiveIdentity) + assert.False(t, got.Status.ActiveController.LastUpdated.IsZero(), "LastUpdated must be stamped") + } +} + +func TestPublishOnce_SecondPassWritesNothing(t *testing.T) { + writes := 0 + c := countingClusterClient(t, &writes, newCluster("worker-1", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, "https://hub-a.example.com:6443") + + require.NoError(t, p.PublishOnce(context.Background())) + assert.Equal(t, 1, writes, "first pass must publish") + + require.NoError(t, p.PublishOnce(context.Background())) + assert.Equal(t, 1, writes, + "a converged pass must not write; comparing LastUpdated would make every tick write to every Cluster CR") +} + +func TestPublishOnce_RepublishesWhenIdentityChanges(t *testing.T) { + existing := newCluster("worker-1", "kubeslice-avesha") + existing.Status.ActiveController = &controllerv1alpha1.ActiveControllerInfo{ + Endpoint: "https://hub-a.example.com:6443", + ActiveIdentity: "hub-a", + LastUpdated: metav1.NewTime(time.Now().Add(-time.Hour)), + } + c := clusterClient(t, existing) + // hub-b has been promoted and now publishes about itself. + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-b"}, "https://hub-b.example.com:6443") + + require.NoError(t, p.PublishOnce(context.Background())) + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + require.NotNil(t, got.Status.ActiveController) + assert.Equal(t, "hub-b", got.Status.ActiveController.ActiveIdentity, + "a promoted hub must overwrite the previous holder's declaration") + assert.Equal(t, "https://hub-b.example.com:6443", got.Status.ActiveController.Endpoint) +} + +func TestPublishOnce_RefusesShippedPlaceholderEndpoint(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, PlaceholderControllerEndpoint) + + require.NoError(t, p.PublishOnce(context.Background()), + "refusing to publish must not be an error — a misconfigured endpoint must not stop the hub reconciling") + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + assert.Nil(t, got.Status.ActiveController, + "publishing the shipped placeholder would advertise an unreachable failover target") +} + +func TestPublishOnce_RefusesEmptyEndpoint(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, "") + + require.NoError(t, p.PublishOnce(context.Background())) + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + assert.Nil(t, got.Status.ActiveController) +} + +// TestPlaceholderMatchesServiceDefault pins the duplicated literal to its source. +// main.go overwrites service.ControllerEndpoint with the flag value at startup, +// so the publisher cannot read the default at runtime and must carry its own copy. +// If the shipped default ever changes, this fails instead of the publisher +// silently starting to advertise it. +func TestPlaceholderMatchesServiceDefault(t *testing.T) { + assert.Equal(t, service.ControllerEndpoint, PlaceholderControllerEndpoint, + "pkg/ha's placeholder copy has drifted from service.ControllerEndpoint's shipped default") +} + +func TestNewActivePublisher_ReadsAndEncodesCABundle(t *testing.T) { + dir := t.TempDir() + caPath := filepath.Join(dir, "ca.crt") + require.NoError(t, os.WriteFile(caPath, []byte("-----BEGIN CERTIFICATE-----\nfake\n"), 0o600)) + + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := NewActivePublisher(c, stubLeadership{leader: true, identity: "hub-a"}, ActivePublisherOptions{ + Endpoint: "https://hub-a.example.com:6443", + CABundlePath: caPath, + Log: testLog(), + }) + require.NoError(t, p.PublishOnce(context.Background())) + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + require.NotNil(t, got.Status.ActiveController) + decoded, err := base64.StdEncoding.DecodeString(got.Status.ActiveController.CABundle) + require.NoError(t, err, "caBundle must be base64-encoded PEM") + assert.Equal(t, "-----BEGIN CERTIFICATE-----\nfake\n", string(decoded)) +} + +func TestNewActivePublisher_PublishesWithoutUnreadableCABundle(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, "https://hub-a.example.com:6443") + + require.NoError(t, p.PublishOnce(context.Background())) + + got := getCluster(t, c, "worker-1", "kubeslice-avesha") + require.NotNil(t, got.Status.ActiveController, + "an unreadable CA bundle must not block publication — endpoint and identity are what select a hub") + assert.Empty(t, got.Status.ActiveController.CABundle) +} + +func TestPublishOnce_ReturnsErrorWhenListFails(t *testing.T) { + base := fake.NewClientBuilder().WithScheme(clusterScheme(t)).Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + return fmt.Errorf("simulated API server down") + }, + }) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, "https://hub-a.example.com:6443") + + assert.Error(t, p.PublishOnce(context.Background()), + "a failed list must surface so the caller can retry, not be swallowed as success") +} + +func TestPublishOnce_ContinuesAfterOneClusterFails(t *testing.T) { + base := fake.NewClientBuilder(). + WithScheme(clusterScheme(t)). + WithObjects(newCluster("worker-1", "kubeslice-avesha"), newCluster("worker-2", "kubeslice-avesha")). + WithStatusSubresource(&controllerv1alpha1.Cluster{}). + Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if obj.GetName() == "worker-1" { + return fmt.Errorf("simulated conflict") + } + return c.Status().Update(ctx, obj) + }, + }) + p := testPublisher(t, c, stubLeadership{leader: true, identity: "hub-a"}, "https://hub-a.example.com:6443") + + assert.Error(t, p.PublishOnce(context.Background()), "the failing cluster must be reported") + + got := getCluster(t, c, "worker-2", "kubeslice-avesha") + assert.NotNil(t, got.Status.ActiveController, + "one cluster failing must not stop the others being published") +} + +// lateLeadership becomes the leader only after IsLeader has been asked a few +// times, standing in for an Active whose Lease renewal lands a second or two +// after start-up. +type lateLeadership struct { + mu sync.Mutex + checks int + after int +} + +func (l *lateLeadership) IsLeader() bool { + l.mu.Lock() + defer l.mu.Unlock() + l.checks++ + return l.checks > l.after +} +func (l *lateLeadership) Identity() string { return "hub-a" } + +// TestStart_PublishesPromptlyWhenLeadershipArrivesLate pins the fix for a defect +// found in live testing: Start's first pass runs before the elector has acquired +// its Lease, so it skipped, and the next attempt was a full publish interval +// away — a fresh Active took 31s to advertise itself. While not the leader the +// loop must poll on the much shorter leadership interval instead. +func TestStart_PublishesPromptlyWhenLeadershipArrivesLate(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := NewActivePublisher(c, &lateLeadership{after: 3}, ActivePublisherOptions{ + Endpoint: "https://hub-a.example.com:6443", + // A publish interval far longer than the test's patience: if the loop + // waits this out before retrying, the test fails. + Interval: time.Hour, + LeadershipPollInterval: 5 * time.Millisecond, + CABundlePath: filepath.Join(t.TempDir(), "absent-ca.crt"), + Log: testLog(), + }) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = p.Start(ctx) }() + + require.Eventually(t, func() bool { + return getCluster(t, c, "worker-1", "kubeslice-avesha").Status.ActiveController != nil + }, 2*time.Second, 5*time.Millisecond, + "a hub that becomes leader after start-up must publish on the leadership poll interval, "+ + "not wait out a full publish interval") +} + +func TestStart_ReturnsNilOnContextCancel(t *testing.T) { + c := clusterClient(t, newCluster("worker-1", "kubeslice-avesha")) + p := NewActivePublisher(c, stubLeadership{leader: true, identity: "hub-a"}, ActivePublisherOptions{ + Endpoint: "https://hub-a.example.com:6443", + Interval: 10 * time.Millisecond, + Log: testLog(), + }) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- p.Start(ctx) }() + + require.Eventually(t, func() bool { + return getCluster(t, c, "worker-1", "kubeslice-avesha").Status.ActiveController != nil + }, time.Second, 5*time.Millisecond, "Start must publish immediately, not wait for the first tick") + + cancel() + select { + case err := <-done: + assert.NoError(t, err, "graceful shutdown must not be reported as an error") + case <-time.After(time.Second): + t.Fatal("Start did not return after context cancellation") + } +} diff --git a/pkg/ha/credential_set_test.go b/pkg/ha/credential_set_test.go new file mode 100644 index 000000000..b36499eb2 --- /dev/null +++ b/pkg/ha/credential_set_test.go @@ -0,0 +1,477 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "encoding/base64" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/cache" + + "github.com/kubeslice/kubeslice-controller/util" +) + +// Secret .data values are base64 in the serialised form these tests build, so +// the fixtures are encoded from readable plaintext instead of being written as +// literals. A bare base64 blob in a file about credentials makes a reviewer +// stop and decode it to satisfy themselves it is not a real token, which is a +// poor thing to put in front of someone reading security-adjacent code. +func b64(plain string) string { return base64.StdEncoding.EncodeToString([]byte(plain)) } + +var ( + activeSignedToken = b64("active-signed-token") + standbyMintedToken = b64("standby-minted-token") + gatewayCertData = b64("cert-data") + oldGatewayCert = b64("old-cert") + newGatewayCert = b64("new-cert") + shortToken = b64("token") +) + +var ( + nsGVK = schema.GroupVersionKind{Version: "v1", Kind: "Namespace"} + secretGVK = schema.GroupVersionKind{Version: "v1", Kind: "Secret"} + saGVK = schema.GroupVersionKind{Version: "v1", Kind: "ServiceAccount"} + roleGVK = schema.GroupVersionKind{Group: groupRBAC, Version: "v1", Kind: "Role"} + rbGVK = schema.GroupVersionKind{Group: groupRBAC, Version: "v1", Kind: "RoleBinding"} +) + +// buildCredentialSyncer is buildSyncer's sibling for the credential set. +func buildCredentialSyncer(t *testing.T, remote *stubRemote) *RemoteSyncer { + t.Helper() + byGVK := make(map[schema.GroupVersionKind]MirroredResource, len(CredentialMirrorSet)) + for _, res := range CredentialMirrorSet { + byGVK[res.GVK] = res + } + return &RemoteSyncer{ + mode: ModeStandby, + localClient: fakeClient(t), + remoteGet: remote.get, + resources: CredentialMirrorSet, + byGVK: byGVK, + workers: 1, + queue: workqueue.NewTypedRateLimitingQueue[syncKey]( + workqueue.NewTypedItemExponentialFailureRateLimiter[syncKey](time.Millisecond, time.Second), + ), + handlerRegistered: map[schema.GroupVersionKind]bool{}, + enqueuedAt: map[syncKey]time.Time{}, + log: testLog(), + } +} + +// registerMirroredNamespace makes ns visible in the stub's Namespace view, +// standing in for a project namespace the label-scoped remote cache mirrors. +func registerMirroredNamespace(remote *stubRemote, ns string) { + key := syncKey{GVK: nsGVK, Name: ns} + remote.objects[key] = newTestUnstructured(nsGVK, "", ns) +} + +func TestCredentialMirrorSet_ShapeAndDefenses(t *testing.T) { + var gvks []schema.GroupVersionKind + for _, res := range CredentialMirrorSet { + gvks = append(gvks, res.GVK) + assert.True(t, res.StripOwnerRefs, + "%s: UID-based ownerReferences never survive a cross-cluster copy, and credential objects are written by actors outside this repo — every row must strip them", res.GVK.Kind) + assert.True(t, res.RequireMirroredNamespace, + "%s: core types exist cluster-wide; every row must be gated on the mirrored-namespace boundary", res.GVK.Kind) + } + assert.ElementsMatch(t, []schema.GroupVersionKind{secretGVK, saGVK, roleGVK, rbGVK}, gvks, + "credential set is Secret/ServiceAccount/Role/RoleBinding only — access_control_service never creates ClusterRole/ClusterRoleBinding, despite ADR #293 Decision 6's broader wording") +} + +func TestIsServiceAccountTokenSecret(t *testing.T) { + saToken := newTestUnstructured(secretGVK, "proj", "sa-token") + require.NoError(t, unstructured.SetNestedField(saToken.Object, string(corev1.SecretTypeServiceAccountToken), "type")) + assert.True(t, isServiceAccountTokenSecret(saToken)) + + opaque := newTestUnstructured(secretGVK, "proj", "gateway-cert") + require.NoError(t, unstructured.SetNestedField(opaque.Object, string(corev1.SecretTypeOpaque), "type")) + assert.False(t, isServiceAccountTokenSecret(opaque)) + + untyped := newTestUnstructured(secretGVK, "proj", "no-type-field") + assert.False(t, isServiceAccountTokenSecret(untyped)) +} + +// TestSanitizeSecret_ReducesTokenSecretToItsShell pins the payload-side half of +// the SA-token contract: the account name survives (it is what tells the +// Standby's token controller which account to mint for), the token bytes and +// the account UID do not. +func TestSanitizeSecret_ReducesTokenSecretToItsShell(t *testing.T) { + saToken := newTestUnstructured(secretGVK, "proj", "kubeslice-rbac-worker-w1") + require.NoError(t, unstructured.SetNestedField(saToken.Object, string(corev1.SecretTypeServiceAccountToken), "type")) + require.NoError(t, unstructured.SetNestedField(saToken.Object, activeSignedToken, "data", corev1.ServiceAccountTokenKey)) + saToken.SetAnnotations(map[string]string{ + corev1.ServiceAccountNameKey: "kubeslice-rbac-worker-w1", + corev1.ServiceAccountUIDKey: "11111111-2222-3333-4444-555555555555", + }) + + sanitizeSecret(saToken) + + _, found, err := unstructured.NestedFieldNoCopy(saToken.Object, "data") + require.NoError(t, err) + assert.False(t, found, + "an Active-signed token is invalid on the Standby; shipping it would mask the absence of a real credential") + assert.NotContains(t, saToken.GetAnnotations(), corev1.ServiceAccountUIDKey, + "a copied UID annotation never matches the mirrored ServiceAccount's fresh UID, and the Standby's token controller deletes the Secret on mismatch") + assert.Equal(t, "kubeslice-rbac-worker-w1", saToken.GetAnnotations()[corev1.ServiceAccountNameKey], + "the account name is the only link between the shell and the mirrored ServiceAccount — it must survive") + + // Other Secret types are none of this function's business. + opaque := newTestUnstructured(secretGVK, "proj", "gateway-cert") + require.NoError(t, unstructured.SetNestedField(opaque.Object, string(corev1.SecretTypeOpaque), "type")) + require.NoError(t, unstructured.SetNestedField(opaque.Object, gatewayCertData, "data", "ovpn.crt")) + sanitizeSecret(opaque) + data, found, err := unstructured.NestedString(opaque.Object, "data", "ovpn.crt") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, gatewayCertData, data) +} + +// TestSanitizeCachedSecret_StripsTokenBytesOnTheWayIntoTheCache covers the +// defence-in-depth layer: with the SA-token field selector gone, Active-minted +// tokens would otherwise be held in this process's memory. Both the typed and +// the unstructured path matter — the syncer reads through the latter. +func TestSanitizeCachedSecret_StripsTokenBytesOnTheWayIntoTheCache(t *testing.T) { + typed := &corev1.Secret{ + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{corev1.ServiceAccountTokenKey: []byte("active-signed-token")}, + } + out, err := sanitizeCachedSecret(typed) + require.NoError(t, err) + assert.Nil(t, out.(*corev1.Secret).Data) + assert.NotNil(t, typed.Data, "the informer's own object must never be mutated in place") + + // The UID annotation deliberately survives into the cache: prune diffs + // against this view, and sanitizeSecret drops it from the payload instead. + typedWithAnnotations := &corev1.Secret{Type: corev1.SecretTypeServiceAccountToken} + typedWithAnnotations.SetAnnotations(map[string]string{corev1.ServiceAccountUIDKey: "abc"}) + typedWithAnnotations.Data = map[string][]byte{corev1.ServiceAccountTokenKey: []byte("t")} + out, err = sanitizeCachedSecret(typedWithAnnotations) + require.NoError(t, err) + assert.Equal(t, "abc", out.(*corev1.Secret).GetAnnotations()[corev1.ServiceAccountUIDKey]) + + unstructuredToken := newTestUnstructured(secretGVK, "proj", "sa-token") + require.NoError(t, unstructured.SetNestedField(unstructuredToken.Object, string(corev1.SecretTypeServiceAccountToken), "type")) + require.NoError(t, unstructured.SetNestedField(unstructuredToken.Object, shortToken, "data", corev1.ServiceAccountTokenKey)) + out, err = sanitizeCachedSecret(unstructuredToken) + require.NoError(t, err) + _, found, err := unstructured.NestedFieldNoCopy(out.(*unstructured.Unstructured).Object, "data") + require.NoError(t, err) + assert.False(t, found) + + // Certificate Secrets and non-Secrets pass through untouched. + opaque := &corev1.Secret{Type: corev1.SecretTypeOpaque, Data: map[string][]byte{"ovpn.crt": []byte("cert")}} + out, err = sanitizeCachedSecret(opaque) + require.NoError(t, err) + assert.Equal(t, []byte("cert"), out.(*corev1.Secret).Data["ovpn.crt"]) + + sa := newTestUnstructured(saGVK, "proj", "kubeslice-rbac-worker-w1") + out, err = sanitizeCachedSecret(sa) + require.NoError(t, err) + assert.Same(t, sa, out) +} + +func TestFullMirrorSet_CombinesBothSetsWithoutCollisions(t *testing.T) { + full := FullMirrorSet() + assert.Len(t, full, len(CRDMirrorSet)+len(CredentialMirrorSet)) + + seen := map[schema.GroupVersionKind]bool{} + for _, res := range full { + assert.False(t, seen[res.GVK], "duplicate mirror row for %s — byGVK keying would silently drop one", res.GVK) + seen[res.GVK] = true + } + assert.True(t, seen[nsGVK]) + assert.True(t, seen[secretGVK]) +} + +func TestReconcileKey_MirrorsOpaqueSecretWithData(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: secretGVK, Namespace: "kubeslice-avesha", Name: "gateway-cert"} + + src := newTestUnstructured(secretGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, string(corev1.SecretTypeOpaque), "type")) + require.NoError(t, unstructured.SetNestedField(src.Object, gatewayCertData, "data", "ovpn.crt")) + + remote := newStubRemote() + remote.objects[key] = src + registerMirroredNamespace(remote, key.Namespace) + s := buildCredentialSyncer(t, remote) + + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, opCreate, op) + + got := getUnstructured(t, s.localClient, key) + assert.Equal(t, LabelValueActive, got.GetLabels()[LabelSyncedFromActive]) + data, found, err := unstructured.NestedString(got.Object, "data", "ovpn.crt") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, gatewayCertData, data, "the mirrored Secret must carry the source's data through unchanged") +} + +// newActiveTokenSecret builds an SA-token Secret as it looks on the Active +// once that cluster's token controller has populated it. +func newActiveTokenSecret(t *testing.T, key syncKey) *unstructured.Unstructured { + t.Helper() + src := newTestUnstructured(secretGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, string(corev1.SecretTypeServiceAccountToken), "type")) + require.NoError(t, unstructured.SetNestedField(src.Object, activeSignedToken, "data", corev1.ServiceAccountTokenKey)) + src.SetAnnotations(map[string]string{ + corev1.ServiceAccountNameKey: key.Name, + corev1.ServiceAccountUIDKey: "11111111-2222-3333-4444-555555555555", + }) + return src +} + +// TestReconcileKey_MirrorsServiceAccountTokenSecretAsShell covers the hub side +// of the worker's dual-hub credential: the Standby has to hold a worker +// credential valid on *itself* before any failover, and it cannot mint one from +// a fenced reconciler. Carrying the empty shell across lets its own token +// controller do it. What must not cross is the Active's token. +func TestReconcileKey_MirrorsServiceAccountTokenSecretAsShell(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: secretGVK, Namespace: "kubeslice-avesha", Name: "kubeslice-rbac-worker-w1"} + + remote := newStubRemote() + remote.objects[key] = newActiveTokenSecret(t, key) + registerMirroredNamespace(remote, key.Namespace) + s := buildCredentialSyncer(t, remote) + + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, opCreate, op) + + got := getUnstructured(t, s.localClient, key) + assert.Equal(t, LabelValueActive, got.GetLabels()[LabelSyncedFromActive]) + assert.Equal(t, string(corev1.SecretTypeServiceAccountToken), got.Object["type"]) + _, found, err := unstructured.NestedFieldNoCopy(got.Object, "data") + require.NoError(t, err) + assert.False(t, found, "an Active-signed token must never land on the Standby") + assert.NotContains(t, got.GetAnnotations(), corev1.ServiceAccountUIDKey) + assert.Equal(t, key.Name, got.GetAnnotations()[corev1.ServiceAccountNameKey], + "without this annotation the Standby's token controller has nothing to mint against") +} + +// TestReconcileKey_NeverOverwritesAMintedTokenSecret is the regression test for +// the failure mode that makes the shell approach non-trivial. The engine's +// update path is an unconditional full write and the remote informer resyncs +// every DefaultInformerResyncPeriod, so without CreateOnly every resync would +// clear the token the Standby's token controller minted, that controller would +// mint a fresh one, and any copy already handed to a worker would stop +// authenticating — on a timer, silently. +func TestReconcileKey_NeverOverwritesAMintedTokenSecret(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: secretGVK, Namespace: "kubeslice-avesha", Name: "kubeslice-rbac-worker-w1"} + + remote := newStubRemote() + remote.objects[key] = newActiveTokenSecret(t, key) + registerMirroredNamespace(remote, key.Namespace) + s := buildCredentialSyncer(t, remote) + + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + require.Equal(t, opCreate, op) + + // Stand in for the Standby's token controller populating the shell. + minted := getUnstructured(t, s.localClient, key) + require.NoError(t, unstructured.SetNestedField(minted.Object, standbyMintedToken, "data", corev1.ServiceAccountTokenKey)) + require.NoError(t, s.localClient.Update(ctx, minted)) + + // A resync delivers the Active's copy again. + op, _, err = s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, mirrorOp(""), op, "a populated token Secret must not be rewritten") + + got := getUnstructured(t, s.localClient, key) + token, found, err := unstructured.NestedString(got.Object, "data", corev1.ServiceAccountTokenKey) + require.NoError(t, err) + require.True(t, found, "the locally minted token must survive a resync") + assert.Equal(t, standbyMintedToken, token) +} + +// TestReconcileKey_StillUpdatesNonTokenSecretsOnResync pins CreateOnly as a +// per-object predicate rather than a property of the whole Secret row: gateway +// certificates must keep converging on the Active's content. +func TestReconcileKey_StillUpdatesNonTokenSecretsOnResync(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: secretGVK, Namespace: "kubeslice-avesha", Name: "gateway-cert"} + + src := newTestUnstructured(secretGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, string(corev1.SecretTypeOpaque), "type")) + require.NoError(t, unstructured.SetNestedField(src.Object, oldGatewayCert, "data", "ovpn.crt")) + + remote := newStubRemote() + remote.objects[key] = src + registerMirroredNamespace(remote, key.Namespace) + s := buildCredentialSyncer(t, remote) + + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + require.Equal(t, opCreate, op) + + rotated := src.DeepCopy() + require.NoError(t, unstructured.SetNestedField(rotated.Object, newGatewayCert, "data", "ovpn.crt")) + remote.objects[key] = rotated + + op, _, err = s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, opUpdate, op) + + got := getUnstructured(t, s.localClient, key) + cert, found, err := unstructured.NestedString(got.Object, "data", "ovpn.crt") + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, newGatewayCert, cert, "certificate rotation on the Active must still reach the Standby") +} + +// TestPruneOnce_LeavesAMintedTokenSecretAlone closes the loop with the prune +// backstop, the other path that can write to a mirrored object. A shell that +// exists on both sides is neither an orphan nor missing, so neither diff +// direction touches it; and if some other round does re-enqueue it, the +// create-only guard still refuses to overwrite the minted token. Either way the +// worker's credential survives. +func TestPruneOnce_LeavesAMintedTokenSecretAlone(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: secretGVK, Namespace: "kubeslice-avesha", Name: "kubeslice-rbac-worker-w1"} + + remote := newStubRemote() + remote.objects[key] = newActiveTokenSecret(t, key) + registerMirroredNamespace(remote, key.Namespace) + s := buildCredentialSyncer(t, remote) + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + require.Equal(t, opCreate, op) + + minted := getUnstructured(t, s.localClient, key) + require.NoError(t, unstructured.SetNestedField(minted.Object, standbyMintedToken, "data", corev1.ServiceAccountTokenKey)) + require.NoError(t, s.localClient.Update(ctx, minted)) + + s.remoteList = stubRemoteList([]syncKey{key}, nil) + s.pruneOnce(ctx) + assert.Equal(t, 0, s.queue.Len(), "a shell present on both sides is neither orphaned nor missing") + + got := getUnstructured(t, s.localClient, key) + token, found, err := unstructured.NestedString(got.Object, "data", corev1.ServiceAccountTokenKey) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, standbyMintedToken, token) +} + +// TestReconcileKey_SkipsCredentialsInUnmirroredNamespaces pins the boundary +// that matters most: a namespace that is not label-mirrored is out of bounds +// no matter what it is named. The concrete case that motivated this (found +// live against a Helm-installed Active hub): under the chart's +// --project-namespace-prefix ("kubeslice-"), the controller's own +// kubeslice-controller namespace looks like a project namespace by name, and +// a name-based rule would have mirrored its webhook TLS key and image-pull +// Secrets onto the Standby. +func TestReconcileKey_SkipsCredentialsInUnmirroredNamespaces(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + gvk schema.GroupVersionKind + ns string + name string + }{ + {secretGVK, "kubeslice-controller", "webhook-server-cert-secret"}, + {secretGVK, "kube-system", "bootstrap-token"}, + {saGVK, "kube-system", "hand-labeled-sa"}, + {roleGVK, "default", "some-role"}, + {rbGVK, "default", "some-rolebinding"}, + } { + key := syncKey{GVK: tc.gvk, Namespace: tc.ns, Name: tc.name} + src := newTestUnstructured(tc.gvk, tc.ns, tc.name) + if tc.gvk == secretGVK { + require.NoError(t, unstructured.SetNestedField(src.Object, string(corev1.SecretTypeOpaque), "type")) + } + + remote := newStubRemote() + remote.objects[key] = src // object visible, namespace deliberately NOT mirrored + s := buildCredentialSyncer(t, remote) + + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, mirrorOp(""), op, "%s %s/%s: unmirrored namespace must mean skip", tc.gvk.Kind, tc.ns, tc.name) + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(tc.gvk) + err = s.localClient.Get(ctx, types.NamespacedName{Namespace: tc.ns, Name: tc.name}, existing) + assert.Error(t, err, "%s %s/%s must not exist on the Standby", tc.gvk.Kind, tc.ns, tc.name) + } +} + +func TestReconcileKey_NamespaceCheckErrorIsRetryable(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: secretGVK, Namespace: "kubeslice-avesha", Name: "gateway-cert"} + + src := newTestUnstructured(secretGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, string(corev1.SecretTypeOpaque), "type")) + + remote := newStubRemote() + remote.objects[key] = src + remote.errs[syncKey{GVK: nsGVK, Name: key.Namespace}] = fmt.Errorf("simulated transient cache failure") + s := buildCredentialSyncer(t, remote) + + _, _, err := s.reconcileKey(ctx, key) + assert.Error(t, err, + "a transient failure reading the namespace must surface as an error (workqueue retry), not as a silent skip") +} + +func TestMirrorCacheByObject_ScopesCredentialInformers(t *testing.T) { + byObject := mirrorCacheByObject() + + // The label-scoped types must match exactly what the controller stamps + // (via ReconcileProjectNamespace and util.GetOwnerLabel) and nothing else. + labeled := labels.Set(util.LabelsKubeSliceController) + for obj, cfg := range byObject { + if _, isSecret := obj.(*corev1.Secret); isSecret { + continue + } + require.NotNil(t, cfg.Label, "%T informer must be label-scoped", obj) + assert.True(t, cfg.Label.Matches(labeled), "%T: selector must match controller-stamped labels", obj) + assert.False(t, cfg.Label.Matches(labels.Set{}), "%T: selector must not match unlabeled objects", obj) + } + + // Secret can be scoped neither way: cert Secrets come from the external + // cert-generator job unlabeled, and the SA-token shells the Standby needs + // rule out the "type" field selector that used to exclude them. It is + // cached cluster-wide instead, with the project-namespace boundary held + // client-side by RequireMirroredNamespace and the token bytes dropped on + // the way in. + var secretCfg cache.ByObject + ok := false + for obj, cfg := range byObject { + if _, isSecret := obj.(*corev1.Secret); isSecret { + secretCfg, ok = cfg, true + } + } + require.True(t, ok, "Secret informer must have a ByObject entry") + assert.Nil(t, secretCfg.Field, + "a type-based field selector cannot admit both SA-token shells and unlabeled certificate Secrets") + assert.Nil(t, secretCfg.Label) + require.NotNil(t, secretCfg.Transform, + "caching every Secret cluster-wide is only acceptable because Active-minted tokens are stripped on ingress") +} diff --git a/pkg/ha/events_test.go b/pkg/ha/events_test.go new file mode 100644 index 000000000..957b28235 --- /dev/null +++ b/pkg/ha/events_test.go @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "testing" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" +) + +// syncFailedEvents lists the HAMirrorSyncFailed events currently on c, in +// key.Namespace. +func syncFailedEvents(t *testing.T, c client.Client, namespace string) []corev1.Event { + t.Helper() + list := &corev1.EventList{} + require.NoError(t, c.List(context.Background(), list, client.InNamespace(namespace))) + var out []corev1.Event + for _, ev := range list.Items { + if ev.Reason == string(ossEvents.EventHAMirrorSyncFailed) { + out = append(out, ev) + } + } + return out +} + +func testEventRecorder(t *testing.T, c client.Client, eventsMap map[events.EventName]*events.EventSchema) events.EventRecorder { + t.Helper() + return events.NewEventRecorder(c, testScheme(t), eventsMap, events.EventRecorderOptions{ + Version: "v1alpha1", + Cluster: "test-cluster", + Component: "controller", + }) +} + +func TestProcessOnce_EmitsHAMirrorSyncFailedOncePerFailureEpisode(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + remote := newStubRemote() + remote.errs[key] = fmt.Errorf("simulated transient read failure") + + s := buildSyncer(t, remote) + eventsClient := fakeClient(t) + s.eventRecorder = testEventRecorder(t, eventsClient, ossEvents.EventsMap) + + // First failure of the episode -> exactly one event. + s.queue.Add(key) + k, _ := s.queue.Get() + s.processOnce(ctx, k) + + got := syncFailedEvents(t, eventsClient, key.Namespace) + require.Len(t, got, 1, "the first mirror failure must surface as a Kubernetes event") + assert.Equal(t, key.Name, got[0].InvolvedObject.Name, "the event must be attached to the object that failed to mirror") + assert.Equal(t, int32(1), got[0].Count) + + // A retry failing within the same episode must not emit again — the + // recorder aggregates by Count, so an ungated second RecordEvent call + // would show up here as Count==2. + k, _ = s.queue.Get() // AddRateLimited's redelivery + s.processOnce(ctx, k) + + got = syncFailedEvents(t, eventsClient, key.Namespace) + require.Len(t, got, 1) + assert.Equal(t, int32(1), got[0].Count, "retries within one failure episode must not re-emit the event") + + // Recovery resets the episode (Forget zeroes NumRequeues)... + remote.mu.Lock() + delete(remote.errs, key) + remote.objects[key] = newTestUnstructured(testGVK, key.Namespace, key.Name) + remote.mu.Unlock() + k, _ = s.queue.Get() + s.processOnce(ctx, k) + require.Len(t, syncFailedEvents(t, eventsClient, key.Namespace), 1, "a successful sync must not emit a failure event") + + // ...so a fresh failure afterwards is a new episode and emits again. + remote.mu.Lock() + remote.errs[key] = fmt.Errorf("simulated second outage") + remote.mu.Unlock() + s.queue.Add(key) + k, _ = s.queue.Get() + s.processOnce(ctx, k) + + got = syncFailedEvents(t, eventsClient, key.Namespace) + require.Len(t, got, 1) + assert.Equal(t, int32(2), got[0].Count, "a new failure episode after recovery must emit again") +} + +func TestProcessOnce_NoRecorderMeansNoEventButStillRetries(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + remote := newStubRemote() + remote.errs[key] = fmt.Errorf("simulated transient read failure") + + s := buildSyncer(t, remote) // eventRecorder deliberately left nil + s.queue.Add(key) + k, _ := s.queue.Get() + s.processOnce(ctx, k) + + // The nil recorder must not panic, and the retry contract is unchanged: + // the key comes back. + k, shutdown := s.queue.Get() + require.False(t, shutdown) + assert.Equal(t, key, k) +} + +// TestHAMirrorSyncFailedEvent_RegisteredInGeneratedMap guards the +// generate-events step this feature depends on: RecordEvent hard-fails for +// any EventName missing from the generated EventsMap, so if the +// config/events/controller.yaml entry (or the generated code) is ever +// reverted, this fails in `go test` rather than silently no-opping at +// runtime. +func TestHAMirrorSyncFailedEvent_RegisteredInGeneratedMap(t *testing.T) { + ctx := context.Background() + require.Contains(t, ossEvents.EventsMap, ossEvents.EventHAMirrorSyncFailed, + "HAMirrorSyncFailed must be present in the generated EventsMap — re-run `make generate-events` if config/events/controller.yaml changed") + + obj := newTestUnstructured(testGVK, "proj-a", "sc-1") + registered := testEventRecorder(t, fakeClient(t), ossEvents.EventsMap) + assert.NoError(t, registered.RecordEvent(ctx, &events.Event{ + Object: obj, + ReportingInstance: "controller", + Name: ossEvents.EventHAMirrorSyncFailed, + }), "recording HAMirrorSyncFailed against the real generated EventsMap must succeed") + + // And the failure mode being guarded against is loud, not silent: + unregistered := testEventRecorder(t, fakeClient(t), map[events.EventName]*events.EventSchema{}) + assert.Error(t, unregistered.RecordEvent(ctx, &events.Event{ + Object: obj, + ReportingInstance: "controller", + Name: ossEvents.EventHAMirrorSyncFailed, + }), "an EventName absent from EventsMap must error, proving a missing generated entry cannot no-op silently") +} diff --git a/pkg/ha/kicker.go b/pkg/ha/kicker.go new file mode 100644 index 000000000..1e80a08a4 --- /dev/null +++ b/pkg/ha/kicker.go @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + "github.com/kubeslice/kubeslice-controller/util" +) + +// The types that have a reconciler and therefore need waking after a +// promotion. This is CRDMirrorSet minus Namespace: the mirror copies namespaces +// so their contents have somewhere to land, but no reconciler owns them, so +// there is nothing to kick. +var ( + GVKProject = gvk(groupController, "Project") + GVKCluster = gvk(groupController, "Cluster") + GVKSliceConfig = gvk(groupController, "SliceConfig") + GVKServiceExportConfig = gvk(groupController, "ServiceExportConfig") + GVKSliceQoSConfig = gvk(groupController, "SliceQoSConfig") + GVKVpnKeyRotation = gvk(groupController, "VpnKeyRotation") + GVKWorkerSliceConfig = gvk(groupWorker, "WorkerSliceConfig") + GVKWorkerSliceGateway = gvk(groupWorker, "WorkerSliceGateway") + GVKWorkerServiceImport = gvk(groupWorker, "WorkerServiceImport") +) + +// ReconciledGVKs returns the nine types the controller reconciles, as a fresh +// slice so callers cannot mutate the package's own view. +func ReconciledGVKs() []schema.GroupVersionKind { + return []schema.GroupVersionKind{ + GVKProject, GVKCluster, GVKSliceConfig, GVKServiceExportConfig, + GVKSliceQoSConfig, GVKVpnKeyRotation, + GVKWorkerSliceConfig, GVKWorkerSliceGateway, GVKWorkerServiceImport, + } +} + +// DefaultKickChannelBuffer is the per-type channel depth. Sized so an ordinary +// hub's whole object set fits without the kick having to block on a consumer +// that has not started draining yet; see Kick for what happens when it does not. +const DefaultKickChannelBuffer = 256 + +// ReconcileKicker re-enqueues every object of every reconciled type after a +// promotion. +// +// It exists because flipping the write fence causes no reconcile at all. The +// fence returns without requeuing, so every request a Standby dropped is +// discarded rather than parked, and nothing fires again until an object changes +// or the informer resyncs — ten hours by default. A promoted hub would sit on +// state it believes it owns and never touch it. Most visibly, the mirror strips +// finalizers by design and only a running reconciler re-adds them, so until +// this runs a delete on the promoted hub skips its cleanup entirely and the +// object simply vanishes. +// +// Note the narrowness of the symptom: an object created on the promoted hub +// after the fence opens reconciles fine without any of this, because it +// generates its own event. Only pre-existing mirrored state stays frozen, which +// is why the gap is easy to miss in a smoke test. +type ReconcileKicker struct { + // channels is one channel per GVK, each wired into that type's controller + // through source.Channel. + // + // One shared channel does not work, and the failure is quiet. Every + // source.Channel starts its own goroutine reading the channel it was given + // and fanning out to its own handler; nine sources over one Go channel means + // nine goroutines competing for each value, so every event goes to exactly + // one arbitrary controller and each type sees a random subset of its own + // objects. In a small test with a couple of objects that looks like it + // works. + channels map[schema.GroupVersionKind]chan event.GenericEvent + + localClient client.Client + log *zap.SugaredLogger +} + +// NewReconcileKicker builds a kicker with one channel per given GVK. +func NewReconcileKicker(localClient client.Client, gvks []schema.GroupVersionKind, log *zap.SugaredLogger) *ReconcileKicker { + if log == nil { + log = util.NewLogger().With("name", "ha-reconcile-kicker") + } + channels := make(map[schema.GroupVersionKind]chan event.GenericEvent, len(gvks)) + for _, gvk := range gvks { + channels[gvk] = make(chan event.GenericEvent, DefaultKickChannelBuffer) + } + return &ReconcileKicker{channels: channels, localClient: localClient, log: log} +} + +// Source returns the channel a controller should read, or nil if this kicker +// does not cover that type. A nil channel is safe to pass on: the caller simply +// registers no extra watch, which leaves that controller exactly as it is today. +func (k *ReconcileKicker) Source(gvk schema.GroupVersionKind) <-chan event.GenericEvent { + if k == nil { + return nil + } + ch, ok := k.channels[gvk] + if !ok { + return nil + } + return ch +} + +// Kick lists every object of every registered type and pushes one event per +// object. It is one-shot and bounded by the size of the cluster's own state: +// there is no steady-state cost, and nothing here runs again until the next +// promotion. +// +// A type whose list fails is reported and the rest still run. Partial coverage +// beats none — the alternative is a promoted hub with nothing reconciled at all +// because one API call failed. +// +// Sends are non-blocking. The consumers are controller-runtime sources, which +// only start draining once the manager is running, and main.go starts the +// promotion path before mgr.Start; a blocking send in that window would hang +// promotion on a channel nobody is reading. A full channel is therefore counted +// and logged rather than waited on. Losing a kick costs a reconcile that would +// have happened anyway on the next change or resync, which is the same position +// the hub is in without this component at all. +func (k *ReconcileKicker) Kick(ctx context.Context) error { + if k == nil || len(k.channels) == 0 { + return nil + } + + var ( + total int + dropped int + failed []string + ) + for gvk, ch := range k.channels { + // Checked explicitly rather than as a select case alongside the send + // below. Both would be ready whenever the channel has room, and select + // chooses randomly among ready cases, so cancellation would only be + // honoured by chance — which is exactly how this was written first, and + // what -shuffle caught. + if err := ctx.Err(); err != nil { + return fmt.Errorf("kicking reconcilers: %w", err) + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) + if err := k.localClient.List(ctx, list); err != nil { + k.log.Errorw("kick: listing local objects failed; this type stays unreconciled until it changes", + "kind", gvk.Kind, "error", err) + failed = append(failed, gvk.Kind) + continue + } + + for i := range list.Items { + if err := ctx.Err(); err != nil { + return fmt.Errorf("kicking reconcilers: %w", err) + } + select { + case ch <- event.GenericEvent{Object: &list.Items[i]}: + total++ + default: + dropped++ + } + } + } + + if dropped > 0 { + k.log.Warnw("kick: some events were dropped because a type's channel was full", + "dropped", dropped, "delivered", total, "buffer", DefaultKickChannelBuffer) + } + k.log.Infow("kick: re-enqueued objects after promotion", + "objects", total, "types", len(k.channels), "dropped", dropped) + + if len(failed) > 0 { + return fmt.Errorf("could not list these types to re-enqueue them: %v", failed) + } + return nil +} diff --git a/pkg/ha/kicker_test.go b/pkg/ha/kicker_test.go new file mode 100644 index 000000000..41772e873 --- /dev/null +++ b/pkg/ha/kicker_test.go @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +func kickObject(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetNamespace(namespace) + u.SetName(name) + return u +} + +// kickClient builds a fake client that can list the mirrored CRD types. +func kickClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder().WithScheme(clusterScheme(t)).WithObjects(objs...).Build() +} + +func drain(ch <-chan event.GenericEvent) []string { + var names []string + for { + select { + case ev := <-ch: + names = append(names, ev.Object.GetName()) + default: + return names + } + } +} + +func TestKick_DeliversOneEventPerObject(t *testing.T) { + c := kickClient(t, + newClusterObj("worker-1", "kubeslice-avesha"), + newClusterObj("worker-2", "kubeslice-avesha"), + ) + k := NewReconcileKicker(c, []schema.GroupVersionKind{GVKCluster}, testLog()) + + require.NoError(t, k.Kick(context.Background())) + + got := drain(k.Source(GVKCluster)) + assert.ElementsMatch(t, []string{"worker-1", "worker-2"}, got, + "every existing object must be re-enqueued; that is the whole point of the kick") +} + +// TestKick_ChannelsAreDisjointPerType is the trap this component exists to +// avoid. Every source.Channel starts its own goroutine reading the channel it +// was handed; nine sources sharing one Go channel would mean nine goroutines +// competing for each value, so each type would receive an arbitrary subset of +// everything and most objects would reach the wrong controller. With a couple +// of objects in a test that still looks like it works, which is what makes it +// dangerous. +func TestKick_ChannelsAreDisjointPerType(t *testing.T) { + c := kickClient(t, + newClusterObj("worker-1", "kubeslice-avesha"), + newProjectObj("avesha", "kubeslice-controller"), + ) + k := NewReconcileKicker(c, []schema.GroupVersionKind{GVKCluster, GVKProject}, testLog()) + + require.NoError(t, k.Kick(context.Background())) + + assert.Equal(t, []string{"worker-1"}, drain(k.Source(GVKCluster)), + "the Cluster channel must carry only Clusters") + assert.Equal(t, []string{"avesha"}, drain(k.Source(GVKProject)), + "the Project channel must carry only Projects") +} + +func TestKick_EmptyClusterIsNotAnError(t *testing.T) { + k := NewReconcileKicker(kickClient(t), []schema.GroupVersionKind{GVKCluster}, testLog()) + + require.NoError(t, k.Kick(context.Background()), "a hub with nothing to reconcile is fine") + assert.Empty(t, drain(k.Source(GVKCluster))) +} + +// TestKick_ContinuesAfterOneTypeFails: partial coverage beats none. The +// alternative is a promoted hub with nothing reconciled at all because one API +// call failed. +func TestKick_ContinuesAfterOneTypeFails(t *testing.T) { + base := fake.NewClientBuilder().WithScheme(clusterScheme(t)). + WithObjects(newProjectObj("avesha", "kubeslice-controller")).Build() + c := interceptor.NewClient(base, interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if list.GetObjectKind().GroupVersionKind().Kind == "ClusterList" { + return fmt.Errorf("simulated list failure") + } + return cl.List(ctx, list, opts...) + }, + }) + k := NewReconcileKicker(c, []schema.GroupVersionKind{GVKCluster, GVKProject}, testLog()) + + err := k.Kick(context.Background()) + require.Error(t, err, "the failing type must be reported") + assert.Contains(t, err.Error(), "Cluster") + assert.Equal(t, []string{"avesha"}, drain(k.Source(GVKProject)), + "one type failing must not stop the others being kicked") +} + +// TestKick_DoesNotBlockOnAFullChannel matters because of when the kick runs. +// Its consumers are controller-runtime sources, which only start draining once +// the manager is running, and main.go starts the promotion path before +// mgr.Start. A blocking send in that window would hang promotion on a channel +// nobody is reading — on a hub that has already taken leadership. +func TestKick_DoesNotBlockOnAFullChannel(t *testing.T) { + var objs []client.Object + for i := 0; i < DefaultKickChannelBuffer+25; i++ { + objs = append(objs, newClusterObj(fmt.Sprintf("worker-%d", i), "kubeslice-avesha")) + } + k := NewReconcileKicker(kickClient(t, objs...), []schema.GroupVersionKind{GVKCluster}, testLog()) + + done := make(chan error, 1) + go func() { done <- k.Kick(context.Background()) }() + + select { + case err := <-done: + assert.NoError(t, err, "dropping events on a full channel is degradation, not failure") + case <-time.After(3 * time.Second): + t.Fatal("Kick blocked on a full channel; nothing drains these until the manager starts, " + + "so this would hang promotion on an already-promoted hub") + } + assert.Len(t, drain(k.Source(GVKCluster)), DefaultKickChannelBuffer, + "the channel should have filled to its buffer and the rest dropped") +} + +// TestKick_RespectsContextCancellation must hold on every run, not most of +// them. Written first with ctx.Done() as a select case beside the send, it +// passed and failed at random: both cases are ready whenever the channel has +// room, and select chooses among ready cases uniformly. -shuffle surfaced it. +func TestKick_RespectsContextCancellation(t *testing.T) { + var objs []client.Object + for i := 0; i < 20; i++ { + objs = append(objs, newClusterObj(fmt.Sprintf("worker-%d", i), "kubeslice-avesha")) + } + k := NewReconcileKicker(kickClient(t, objs...), []schema.GroupVersionKind{GVKCluster}, testLog()) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + for i := 0; i < 50; i++ { + require.Error(t, k.Kick(ctx), + "a cancelled context must abort the kick deterministically, not on a coin flip") + } + assert.Empty(t, drain(k.Source(GVKCluster)), + "and nothing may have been delivered after cancellation") +} + +func TestSource_UnknownTypeIsNil(t *testing.T) { + k := NewReconcileKicker(kickClient(t), []schema.GroupVersionKind{GVKCluster}, testLog()) + assert.Nil(t, k.Source(GVKProject), + "a type the kicker does not cover must yield nil, so the caller simply registers no watch") +} + +// TestNilKicker_IsSafe covers the standalone path: main.go builds a kicker +// unconditionally, but a nil one must behave as "no kick" rather than panic. +func TestNilKicker_IsSafe(t *testing.T) { + var k *ReconcileKicker + assert.Nil(t, k.Source(GVKCluster)) + assert.NoError(t, k.Kick(context.Background())) +} + +func TestReconciledGVKs_CoversEveryReconciledTypeAndNothingElse(t *testing.T) { + got := ReconciledGVKs() + assert.Len(t, got, 9, "there are nine reconcilers; each needs a channel") + + // Namespace is mirrored so contents have somewhere to land, but no + // reconciler owns it, so kicking it would deliver events nothing handles. + for _, g := range got { + assert.NotEqual(t, "Namespace", g.Kind) + } + + // Everything kicked must be something the mirror actually maintains, + // otherwise a promoted hub would be kicking objects it never received. + mirrored := map[schema.GroupVersionKind]bool{} + for _, res := range CRDMirrorSet { + mirrored[res.GVK] = true + } + for _, g := range got { + assert.True(t, mirrored[g], "%s is kicked but not mirrored", g.Kind) + } + + // And the returned slice must be a copy. + got[0] = schema.GroupVersionKind{Kind: "Tampered"} + assert.NotEqual(t, "Tampered", ReconciledGVKs()[0].Kind) +} + +func newClusterObj(name, namespace string) client.Object { + return kickObject(GVKCluster, namespace, name) +} + +func newProjectObj(name, namespace string) client.Object { + return kickObject(GVKProject, namespace, name) +} diff --git a/pkg/ha/leader_elector.go b/pkg/ha/leader_elector.go new file mode 100644 index 000000000..0ade1674a --- /dev/null +++ b/pkg/ha/leader_elector.go @@ -0,0 +1,510 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "os" + "sync/atomic" + "time" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "go.uber.org/zap" + coordinationv1 "k8s.io/api/coordination/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" +) + +// Default Lease coordinates and timings. These mirror the ADR (#293) defaults +// and are overridable through Options / controller flags. +const ( + DefaultLeaseName = "kubeslice-controller-ha" + DefaultLeaseNamespace = "kubeslice-controller" + DefaultLeaseDuration = 15 * time.Second + DefaultRenewDeadline = 10 * time.Second + DefaultRetryPeriod = 2 * time.Second + DefaultPaddingSeconds = 5 * time.Second + + // DefaultPromotionDialTimeout bounds every read a Standby makes of a Lease + // over the network: each periodic poll of the Active's Lease, the + // self-health check against its own API server, and the final dial. All + // must be bounded — main.go builds the remote client with a plain uncached + // client.New and no timeout, so a read from a black-holed API server blocks + // until the OS TCP timeout, which is minutes and far outside the failover + // budget. The periodic poll matters most: the watch loop calls it + // synchronously, so a blocked read stalls detection entirely. + DefaultPromotionDialTimeout = 5 * time.Second + + // DefaultPromotionGracePeriod bounds each step of the promotion sequence + // that waits on another component: stopping the mirror, publishing + // status.activeController, re-enqueuing objects, and emitting the event. + // Expiry aborts the promotion only for the mirror stop, where proceeding + // would mean two writers; the rest log and continue, because a hub that + // cannot publish or emit is still a better Active than none. Distinct from + // PaddingSeconds, which is a detection threshold and has nothing to do with + // sequencing. + DefaultPromotionGracePeriod = 10 * time.Second +) + +// Options configures a ClusterLeaderElector. Zero-valued fields fall back to the +// Default* constants (or the OS hostname, for Identity; the downward-API +// KUBESLICE_CONTROLLER_MANAGER_NAMESPACE env var, for LeaseNamespace). +type Options struct { + Mode HAMode + Identity string + LeaseName string + // LeaseNamespace, if empty, defaults to KUBESLICE_CONTROLLER_MANAGER_NAMESPACE + // (the controller's own namespace, injected via the downward API) so the + // Lease always lands where the leader-election Role grants access to it, + // regardless of which namespace the controller is actually deployed into. + // Only falls back to DefaultLeaseNamespace when that env var is unset too + // (e.g. running outside a pod). + LeaseNamespace string + LeaseDuration time.Duration + RenewDeadline time.Duration + RetryPeriod time.Duration + PaddingSeconds time.Duration + // PromotionDialTimeout bounds every networked Lease read: the periodic poll + // of the Active's Lease and both pre-promotion guard reads. + PromotionDialTimeout time.Duration + // PromotionGracePeriod bounds each promotion step that waits on another + // component. + PromotionGracePeriod time.Duration + // EventRecorder, if set, enables the HA lifecycle Events of issue #298: + // BecameActive / BecameStandby at start-up, LeadershipLost when an Active + // gives up its Lease, PromotionAborted when a Standby declines to take over. + // Optional — nil records nothing. + EventRecorder events.EventRecorder + Log *zap.SugaredLogger +} + +// ClusterLeaderElector coordinates leadership between two hub clusters. Unlike +// controller-runtime's in-cluster --leader-elect (which coordinates pods sharing +// one API server), a Standby elector reads the Active's Lease across a cluster +// boundary through remoteClient. See ADR #293. +type ClusterLeaderElector struct { + localClient client.Client // own cluster — create and renew the Lease + remoteClient client.Client // Standby only — read the Active's Lease (may be nil otherwise) + + // mode is an atomic.Value holding an HAMode, not a plain field, because + // promotion mutates it from its own goroutine while Mode(), StartLeaseRenewal + // and WatchRemoteLease read it from theirs — a plain field would be a data + // race, and -race would rightly say so. + mode atomic.Value + identity string + leaseName string + leaseNS string + + // promoting is held for the whole promotion sequence. IsLeader() reports + // false while it is set, regardless of isLeader, so the write fence stays + // shut from the first step until the last — see promote(). + promoting atomic.Bool + + // hooks are promotion's effects outside the elector. Injected so pkg/ha + // stays independent of the mirror, the publisher and the manager. + hooks PromotionHooks + + // eventRecorder records the lifecycle Events of issue #298 — the mode this + // hub started in, a lost leadership, an abandoned promotion. Optional: nil + // disables them and changes nothing else, which is what keeps every existing + // test constructing an elector without one. + // + // Held directly rather than injected as a hook, unlike EmitPromotedEvent. + // The distinction is which object the Event hangs off. Promotion's Event + // attaches to the Lease it has just acquired, so only promote() can supply + // it; these three attach to the Lease as an identifier rather than as an + // object, which the elector can name unaided from leaseName and leaseNS. + // RemoteSyncer already takes a recorder the same way. + eventRecorder events.EventRecorder + + leaseDuration time.Duration + renewDeadline time.Duration + retryPeriod time.Duration + padding time.Duration + + promotionDialTimeout time.Duration + promotionGracePeriod time.Duration + + // isLeader is the single source of truth read by IsLeader(). The background + // renewal loop keeps it current, so readers never touch the API server. + isLeader atomic.Bool + // lastRenew is the time of the last successful Lease renewal. It is written + // once by promote() when it takes the Lease, before it starts the renewal + // goroutine, and from then on only that goroutine reads and writes it — so + // there is exactly one writer at any time and no lock is needed. + lastRenew time.Time + + // lastSeenLease is the newest Lease successfully read from the Active hub, + // and nil until the very first successful read. It is the whole of the + // promotion trigger (issue #297). + // + // A failed read deliberately leaves it untouched rather than clearing it or + // treating the failure as health. "The Active's controller stopped renewing" + // and "the Active's API server stopped answering" are the same event from + // here — in both, the newest proof of life this hub holds stops advancing — + // so a retained stale Lease ages on its own against a moving clock and one + // comparison covers both. Before this, a read failure reported "not stale", + // which made the loss of an entire hub undetectable. + // + // Only WatchRemoteLease's single goroutine touches this and lastGoodRead. + lastSeenLease *coordinationv1.Lease + // lastGoodRead is the local wall-clock time of that read. Not used by the + // verdict, which anchors on the Lease's own renewTime; carried so an + // optional local-only staleness floor stays available without a redesign + // if clock skew between hubs ever becomes a practical problem. + lastGoodRead time.Time + + log *zap.SugaredLogger +} + +// NewClusterLeaderElector builds an elector. local is a client to this +// controller's own cluster; remote is a client to the Active hub and is required +// only in Standby mode (it may be nil otherwise). Everything else is passed +// through Options, because the Lease timings are operator-configurable flags. +func NewClusterLeaderElector(local, remote client.Client, opts Options) *ClusterLeaderElector { + if opts.Mode == "" { + opts.Mode = ModeStandalone + } + if opts.LeaseName == "" { + opts.LeaseName = DefaultLeaseName + } + if opts.LeaseNamespace == "" { + if ns := os.Getenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE"); ns != "" { + opts.LeaseNamespace = ns + } else { + opts.LeaseNamespace = DefaultLeaseNamespace + } + } + if opts.LeaseDuration == 0 { + opts.LeaseDuration = DefaultLeaseDuration + } + if opts.RenewDeadline == 0 { + opts.RenewDeadline = DefaultRenewDeadline + } + if opts.RetryPeriod == 0 { + opts.RetryPeriod = DefaultRetryPeriod + } + if opts.PaddingSeconds == 0 { + opts.PaddingSeconds = DefaultPaddingSeconds + } + if opts.PromotionDialTimeout == 0 { + opts.PromotionDialTimeout = DefaultPromotionDialTimeout + } + if opts.PromotionGracePeriod == 0 { + opts.PromotionGracePeriod = DefaultPromotionGracePeriod + } + if opts.Identity == "" { + if hostname, err := os.Hostname(); err == nil { + opts.Identity = hostname + } else { + opts.Identity = "kubeslice-controller" + } + } + if opts.Log == nil { + opts.Log = util.NewLogger().With("name", "ha-leader-elector") + } + + e := &ClusterLeaderElector{ + localClient: local, + remoteClient: remote, + identity: opts.Identity, + leaseName: opts.LeaseName, + leaseNS: opts.LeaseNamespace, + leaseDuration: opts.LeaseDuration, + renewDeadline: opts.RenewDeadline, + retryPeriod: opts.RetryPeriod, + padding: opts.PaddingSeconds, + promotionDialTimeout: opts.PromotionDialTimeout, + promotionGracePeriod: opts.PromotionGracePeriod, + eventRecorder: opts.EventRecorder, + log: opts.Log, + } + e.mode.Store(opts.Mode) + // Standalone is always the leader: no Lease, no remote watch — identical to + // the controller's behaviour before HA (the no-regression guarantee). + if opts.Mode == ModeStandalone { + e.isLeader.Store(true) + } + + // Publish the two gauges whose value at 0 is the alertable condition, so the + // series exist from start-up rather than appearing at the first transition. + // This matters more than it looks: a Standby that never promotes never calls + // setLeader, and a Standby that never reads the Active never arms, so on the + // exact hubs an operator most needs to see these, nothing would ever create + // the series and `ha_leader_status == 0` would match no rows at all. + // + // Every other gauge in metrics.go stays deliberately unset until it has a + // real value — see the note there on why a zeroed timestamp is worse than a + // missing one. + haLeaderStatus.Set(boolGauge(e.isLeader.Load())) + if opts.Mode == ModeStandby { + haArmed.WithLabelValues(string(ModeStandby)).Set(0) + } + return e +} + +// IsLeader reports whether this controller may perform mutating reconciles right +// now. It reads an in-memory flag kept current by the background loops, so it is +// cheap enough to call at the top of every Reconcile. The value reflects live +// leadership (refreshed every retryPeriod), never a value frozen at startup. +// It also reports false for the whole of a promotion sequence, regardless of +// isLeader: steps 0 and 8 of promote() bracket the sequence with the promoting +// latch, so the write fence stays shut until the new Active has stopped the +// mirror, taken its Lease and published who it is. That is what gives "the +// reconcilers are not live until promotion finishes" real teeth without any +// external status surface. +func (e *ClusterLeaderElector) IsLeader() bool { + return e.isLeader.Load() && !e.promoting.Load() +} + +// Mode returns the current HA mode. It is read live rather than captured at +// startup, because promotion changes it. +func (e *ClusterLeaderElector) Mode() HAMode { + mode, _ := e.mode.Load().(HAMode) + return mode +} + +// setMode swaps the running mode. Only promote() calls it. +func (e *ClusterLeaderElector) setMode(mode HAMode) { + e.mode.Store(mode) + e.log.Infow("HA mode changed", "mode", mode, "identity", e.identity) +} + +// SetPromotionHooks installs the effects promotion has outside the elector. +// Called from main.go once the mirror, publisher and manager exist — they are +// constructed after the elector, so they cannot be constructor arguments. +// +// A Standby with no hooks still promotes correctly in the narrow sense (it +// takes the Lease and opens the fence); the hooks are what make the promotion +// safe and complete. +func (e *ClusterLeaderElector) SetPromotionHooks(hooks PromotionHooks) { + e.hooks = hooks +} + +// Identity returns this instance's Lease holder identity. +func (e *ClusterLeaderElector) Identity() string { + return e.identity +} + +// StartLeaseRenewal runs the Active's renewal loop until ctx is cancelled. It is +// a no-op in any other mode. It renews immediately, then every retryPeriod. +func (e *ClusterLeaderElector) StartLeaseRenewal(ctx context.Context) error { + if e.Mode() != ModeActive { + e.log.Infow("lease renewal not started; not in active mode", "mode", e.Mode()) + return nil + } + e.log.Infow("starting lease renewal", + "lease", e.leaseName, "namespace", e.leaseNS, "identity", e.identity, + "leaseDuration", e.leaseDuration, "renewDeadline", e.renewDeadline, "retryPeriod", e.retryPeriod) + + _ = e.renewOnce(ctx) + ticker := time.NewTicker(e.retryPeriod) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + e.setLeader(false) + e.log.Infow("lease renewal stopped", "reason", ctx.Err()) + return nil + case <-ticker.C: + _ = e.renewOnce(ctx) + } + } +} + +// renewOnce performs a single acquire/renew attempt and updates leadership state. +// On success it (re)acquires leadership. On failure it keeps leadership until +// renewDeadline elapses without any successful renewal, then releases it — the +// natural-fencing behaviour ADR #293 relies on: a dead API server means no renewal and +// therefore no writes. +func (e *ClusterLeaderElector) renewOnce(ctx context.Context) error { + if _, err := acquireOrRenewLease(ctx, e.localClient, e.leaseName, e.leaseNS, e.identity, e.leaseDuration); err != nil { + haLeaseRenewErrorsTotal.Inc() + switch { + case e.lastRenew.IsZero(): + e.log.Warnw("failed to acquire lease", "error", err) + case time.Since(e.lastRenew) > e.renewDeadline: + e.log.Warnw("failed to renew lease within renew deadline; releasing leadership", + "error", err, "renewDeadline", e.renewDeadline, "sinceLastRenew", time.Since(e.lastRenew)) + // Emitted here rather than inside setLeader, and only on this branch. + // setLeader's other caller for a false value is StartLeaseRenewal's + // ctx.Done path — an ordinary graceful shutdown, where a Warning + // Event would be pure noise and the write would in any case be racing + // the pod's own termination. This branch is the one issue #298 + // describes: renewal has failed for longer than renewDeadline and + // leadership is being given up while the process keeps running. + e.emitLifecycleEvent(ctx, ossEvents.EventHALeadershipLost) + e.setLeader(false) + default: + e.log.Warnw("failed to renew lease; still within renew deadline, keeping leadership", "error", err) + } + return err + } + e.lastRenew = time.Now() + haLeaseLastRenewTime.WithLabelValues(string(ModeActive)).Set(float64(e.lastRenew.Unix())) + e.setLeader(true) + return nil +} + +// WatchRemoteLease runs the Standby's watch loop until ctx is cancelled. It +// reads the Active's Lease every retryPeriod and, once that Lease has aged past +// leaseDuration + padding, runs the promotion sequence (issue #297). +// +// It returns as soon as promotion succeeds: this hub is an Active now, and +// there is no longer any Active to watch. Promotion has already started the +// renewal loop that replaces it. +func (e *ClusterLeaderElector) WatchRemoteLease(ctx context.Context) error { + if e.Mode() != ModeStandby { + e.log.Infow("remote lease watch not started; not in standby mode", "mode", e.Mode()) + return nil + } + if e.remoteClient == nil { + return fmt.Errorf("standby mode requires a remote client to the active hub") + } + e.log.Infow("watching active hub lease", "lease", e.leaseName, "namespace", e.leaseNS) + + ticker := time.NewTicker(e.retryPeriod) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + e.log.Infow("remote lease watch stopped", "reason", ctx.Err()) + return nil + case <-ticker.C: + candidate, _ := e.checkRemoteLeaseOnce(ctx) + if !candidate { + continue + } + promoted, err := e.promote(ctx) + if err != nil { + // Stay a Standby and try again next tick. Every failure path in + // promote() leaves the hub exactly as it was — still fenced, + // still armed — so retrying is safe rather than merely tolerable. + e.log.Errorw("promotion attempt failed; remaining standby", "error", err) + continue + } + if promoted { + e.log.Infow("remote lease watch stopping; this hub is now the active") + return nil + } + } + } +} + +// checkRemoteLeaseOnce reads the Active's Lease once, updates the cached view, +// and reports whether this hub is now a promotion *candidate*. It never changes +// leadership itself — the guards and the promotion sequence are separate, so +// that "we think the Active is gone" and "we took over" stay independently +// testable. +// +// err is the read error, returned for logging and tests; the verdict does not +// depend on it. A read that fails is not evidence of health, it is the absence +// of new evidence — see lastSeenLease. +func (e *ClusterLeaderElector) checkRemoteLeaseOnce(ctx context.Context) (candidate bool, err error) { + // Bounded, for the same reason the guards' reads are. The watch loop calls + // this synchronously, so a read that blocks blocks the loop — and while it is + // blocked no staleness is evaluated at all. main.go builds the remote client + // with a plain uncached client.New and no timeout, so an API server that + // accepts the connection and then stops answering leaves the read hanging + // until the transport gives up: minutes against a black-holed host. + // + // This is not hypothetical. Live-testing an Active whose API server was shut + // down showed a single read blocking for ~12s, with no staleness evaluated in + // the whole window, before the connection finally broke. That was a graceful + // container shutdown; a powered-off node or a dropped-packet partition has + // nothing to break the connection at all, and the failover budget would be + // blown by an unbounded wait rather than by the detection rule. + readCtx, cancel := context.WithTimeout(ctx, e.promotionDialTimeout) + defer cancel() + + lease, err := getLease(readCtx, e.remoteClient, e.leaseName, e.leaseNS) + if err != nil { + haRemoteLeaseReadsTotal.WithLabelValues(readResultError).Inc() + e.log.Warnw("could not read active hub lease; retaining last known view", + "error", err, "armed", e.lastSeenLease != nil) + } else { + haRemoteLeaseReadsTotal.WithLabelValues(readResultOK).Inc() + e.lastSeenLease = lease + e.lastGoodRead = time.Now() + } + haArmed.WithLabelValues(string(ModeStandby)).Set(boolGauge(e.lastSeenLease != nil)) + // Deliberately outside the else: the age is published on failed reads too, + // and that is the whole value of it as a leading indicator. A retained stale + // Lease ageing against a moving clock is exactly how this loop models "no new + // evidence of life", so the gauge climbs through an outage rather than + // freezing at the last good value and looking healthy. + if age, ok := remoteLeaseAge(e.lastSeenLease, time.Now()); ok { + haRemoteLeaseAgeSeconds.WithLabelValues(string(ModeStandby)).Set(age.Seconds()) + } + + // This nil check MUST stay a separate statement and must never be folded + // into the isLeaseStale call below. isLeaseStale(nil, ...) returns TRUE + // (lease.go) — correct for its original caller, where a Lease that does not + // exist on your own cluster is stale and should be created. Here it would + // mean a Standby that has never once read the Active's Lease concludes the + // Active is dead and promotes on its very first tick: a broken kubeconfig, + // a missing RBAC grant or a mistyped namespace would each become a + // guaranteed split brain. TestNeverArmed_NeverBecomesCandidate pins this. + if e.lastSeenLease == nil { + e.log.Warnw("the active hub's lease has never been read successfully; refusing to consider promotion", + "lease", e.leaseName, "namespace", e.leaseNS, + "hint", "check --ha-active-kubeconfig, RBAC for coordination.k8s.io/leases, and the lease namespace") + return false, err + } + + if !isLeaseStale(e.lastSeenLease, e.padding, time.Now()) { + e.log.Debugw("active hub lease is fresh", + "lease", e.leaseName, "holder", leaseHolder(e.lastSeenLease), "renewTime", leaseRenewStr(e.lastSeenLease)) + return false, err + } + + e.log.Warnw("active hub lease is STALE; evaluating promotion", + "lease", e.leaseName, "holder", leaseHolder(e.lastSeenLease), + "renewTime", leaseRenewStr(e.lastSeenLease), "readable", err == nil) + return true, err +} + +// setLeader updates the leadership flag and logs LeadershipAcquired / +// LeadershipLost only on an actual transition. +func (e *ClusterLeaderElector) setLeader(leader bool) { + if e.isLeader.Swap(leader) == leader { + return + } + haLeaderStatus.Set(boolGauge(leader)) + if leader { + e.log.Infow("LeadershipAcquired", "identity", e.identity, "lease", e.leaseName) + } else { + e.log.Infow("LeadershipLost", "identity", e.identity, "lease", e.leaseName) + } +} + +func leaseHolder(lease *coordinationv1.Lease) string { + if lease == nil || lease.Spec.HolderIdentity == nil { + return "" + } + return *lease.Spec.HolderIdentity +} + +func leaseRenewStr(lease *coordinationv1.Lease) string { + if lease == nil || lease.Spec.RenewTime == nil { + return "" + } + return lease.Spec.RenewTime.String() +} diff --git a/pkg/ha/leader_elector_test.go b/pkg/ha/leader_elector_test.go new file mode 100644 index 000000000..4715aeaa7 --- /dev/null +++ b/pkg/ha/leader_elector_test.go @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestNewClusterLeaderElector_StandaloneIsAlwaysLeader(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: ModeStandalone, Log: testLog()}) + assert.True(t, e.IsLeader(), "standalone must be leader") + assert.Equal(t, ModeStandalone, e.Mode()) +} + +func TestNewClusterLeaderElector_DefaultsToStandalone(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Log: testLog()}) + assert.Equal(t, ModeStandalone, e.Mode(), "empty mode must default to standalone (no regression)") + assert.True(t, e.IsLeader()) +} + +func TestNewClusterLeaderElector_LeaseNamespacePrefersDownwardAPIEnvVar(t *testing.T) { + t.Setenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE", "kubeslice-avesha") + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Log: testLog()}) + assert.Equal(t, "kubeslice-avesha", e.leaseNS, + "an empty LeaseNamespace must prefer the controller's own runtime namespace over the hard-coded default") +} + +func TestNewClusterLeaderElector_LeaseNamespaceFallsBackWhenEnvVarUnset(t *testing.T) { + t.Setenv("KUBESLICE_CONTROLLER_MANAGER_NAMESPACE", "") + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Log: testLog()}) + assert.Equal(t, DefaultLeaseNamespace, e.leaseNS, + "with no env var and no explicit Options.LeaseNamespace, must fall back to DefaultLeaseNamespace") +} + +func TestActive_BecomesLeaderAfterRenew(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: ModeActive, Log: testLog()}) + assert.False(t, e.IsLeader(), "active is not leader until it renews") + require.NoError(t, e.renewOnce(context.Background())) + assert.True(t, e.IsLeader(), "active should hold leadership after a successful renew") +} + +func TestActive_LosesLeadershipAfterRenewDeadline(t *testing.T) { + e := NewClusterLeaderElector(failingWriteClient(t), nil, Options{ + Mode: ModeActive, + RenewDeadline: 10 * time.Millisecond, + Log: testLog(), + }) + // Simulate having been the leader, with the last successful renew well past + // the renew deadline. + e.isLeader.Store(true) + e.lastRenew = time.Now().Add(-time.Hour) + + err := e.renewOnce(context.Background()) + require.Error(t, err) + assert.False(t, e.IsLeader(), "leadership must drop once renewDeadline is exceeded (natural fencing)") +} + +func TestStandby_NeverLeaderEvenWhenLeaseStale(t *testing.T) { + // A fresh lease on the remote: the standby stays a standby. + freshRemote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + e := NewClusterLeaderElector(fakeClient(t), freshRemote, Options{Mode: ModeStandby, Log: testLog()}) + assert.False(t, e.IsLeader()) + stale, err := e.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + assert.False(t, stale) + assert.False(t, e.IsLeader()) + + // A stale lease on the remote: #294 detects staleness but must NOT promote. + staleRemote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour))) + e2 := NewClusterLeaderElector(fakeClient(t), staleRemote, Options{Mode: ModeStandby, Log: testLog()}) + stale2, err := e2.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + assert.True(t, stale2, "old renewTime should read as stale") + assert.False(t, e2.IsLeader(), "standby must not promote in #294 (promotion is #297)") +} + +func TestCheckRemoteLeaseOnce_PropagatesGetError(t *testing.T) { + remote := fakeClient(t) // the Active's lease is not present on the remote + e := NewClusterLeaderElector(fakeClient(t), remote, Options{Mode: ModeStandby, Log: testLog()}) + + stale, err := e.checkRemoteLeaseOnce(context.Background()) + require.Error(t, err, "a missing remote lease must surface as an error, not silently report fresh") + assert.False(t, stale) +} + +func TestRenewOnce_KeepsLeadershipWithinRenewDeadline(t *testing.T) { + e := NewClusterLeaderElector(failingWriteClient(t), nil, Options{ + Mode: ModeActive, + RenewDeadline: time.Hour, + Log: testLog(), + }) + e.isLeader.Store(true) + e.lastRenew = time.Now() // just renewed, well within the deadline + + err := e.renewOnce(context.Background()) + require.Error(t, err, "a failed renew attempt must still surface an error to the caller") + assert.True(t, e.IsLeader(), "leadership must be kept while still within renewDeadline (transient failure)") +} + +func TestSetLeader_LogsOnlyOnTransition(t *testing.T) { + core, logs := observer.New(zapcore.InfoLevel) + log := zap.New(core).Sugar() + + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: ModeActive, Identity: "hub-a", Log: log}) + + e.setLeader(true) + e.setLeader(true) // no transition; must not log again + e.setLeader(false) + e.setLeader(false) // no transition; must not log again + + assert.Equal(t, 1, logs.FilterMessage("LeadershipAcquired").Len(), + "LeadershipAcquired must be logged exactly once per actual transition") + assert.Equal(t, 1, logs.FilterMessage("LeadershipLost").Len(), + "LeadershipLost must be logged exactly once per actual transition") +} + +func TestStartLeaseRenewal_NoopWhenNotActive(t *testing.T) { + for _, mode := range []HAMode{ModeStandby, ModeStandalone} { + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t), Options{Mode: mode, Log: testLog()}) + err := e.StartLeaseRenewal(context.Background()) + assert.NoError(t, err, "StartLeaseRenewal must be a no-op outside active mode") + } +} + +func TestStartLeaseRenewal_ReturnsNilOnContextCancellation(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{ + Mode: ModeActive, + RetryPeriod: 20 * time.Millisecond, + Log: testLog(), + }) + ctx, cancel := context.WithCancel(context.Background()) + + errCh := make(chan error, 1) + go func() { errCh <- e.StartLeaseRenewal(ctx) }() + + require.Eventually(t, e.IsLeader, time.Second, 5*time.Millisecond, + "elector should acquire leadership before shutdown") + + cancel() + + select { + case err := <-errCh: + assert.NoError(t, err, "a graceful shutdown (context cancellation) must not be reported as an error") + case <-time.After(time.Second): + t.Fatal("StartLeaseRenewal did not return after context cancellation") + } + assert.False(t, e.IsLeader(), "leadership must be released on shutdown") +} + +func TestWatchRemoteLease_NoopWhenNotStandby(t *testing.T) { + for _, mode := range []HAMode{ModeActive, ModeStandalone} { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: mode, Log: testLog()}) + err := e.WatchRemoteLease(context.Background()) + assert.NoError(t, err, "WatchRemoteLease must be a no-op outside standby mode") + } +} + +func TestWatchRemoteLease_RequiresRemoteClientInStandbyMode(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: ModeStandby, Log: testLog()}) + err := e.WatchRemoteLease(context.Background()) + assert.Error(t, err, "standby mode without a remote client must fail fast instead of watching nothing") +} + +func TestWatchRemoteLease_ReturnsNilOnContextCancellation(t *testing.T) { + remote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + e := NewClusterLeaderElector(fakeClient(t), remote, Options{ + Mode: ModeStandby, + RetryPeriod: 20 * time.Millisecond, + Log: testLog(), + }) + ctx, cancel := context.WithCancel(context.Background()) + + errCh := make(chan error, 1) + go func() { errCh <- e.WatchRemoteLease(ctx) }() + + time.Sleep(50 * time.Millisecond) // let at least one watch tick fire + cancel() + + select { + case err := <-errCh: + assert.NoError(t, err, "a graceful shutdown (context cancellation) must not be reported as an error") + case <-time.After(time.Second): + t.Fatal("WatchRemoteLease did not return after context cancellation") + } +} + +// --- issue #297: detection --------------------------------------------------- + +// TestNeverArmed_NeverBecomesCandidate is the regression test for the single +// most dangerous mistake available in this file. isLeaseStale(nil, ...) returns +// TRUE, so folding the lastSeenLease nil check into that call would make a +// Standby that has never once read the Active's Lease promote itself on its +// first tick — turning a broken kubeconfig or a missing RBAC grant into a +// guaranteed split brain. If someone "simplifies" the two conditions into one, +// this test fails. +func TestNeverArmed_NeverBecomesCandidate(t *testing.T) { + // A remote client with no Lease at all: every read fails, so the elector + // never arms. + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + for i := 0; i < 5; i++ { + candidate, err := e.checkRemoteLeaseOnce(context.Background()) + require.Error(t, err, "the read must genuinely be failing for this test to mean anything") + assert.False(t, candidate, + "an elector that has never read the Active's lease must never become a promotion candidate, "+ + "however long it waits — that is a configuration failure, not a dead Active") + } + assert.Nil(t, e.lastSeenLease, "a failed read must not populate the cached lease") +} + +// TestUnreadableLease_RetainsCacheAndGoesStale is the whole point of #297's +// detection change: an Active whose API server dies (reads fail) must be +// detected exactly like an Active whose pod dies (reads succeed, renewTime +// frozen). Before this, a read failure reported "not stale" forever and the +// loss of an entire hub was undetectable. +func TestUnreadableLease_RetainsCacheAndGoesStale(t *testing.T) { + // Arm against a fresh lease. + fresh := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now()) + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t, fresh), Options{Mode: ModeStandby, Log: testLog()}) + + candidate, err := e.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + require.False(t, candidate, "a fresh lease is not a candidate") + require.NotNil(t, e.lastSeenLease, "a successful read must arm the elector") + + // Now the Active's API server dies: every subsequent read fails. + e.remoteClient = fakeClient(t) + + candidate, err = e.checkRemoteLeaseOnce(context.Background()) + require.Error(t, err) + assert.False(t, candidate, "the retained lease is still fresh; a failed read alone proves nothing") + assert.NotNil(t, e.lastSeenLease, "a failed read must retain the last good view, not clear it") + + // Age the retained view past leaseDuration + padding. This is what a real + // clock does on its own while reads keep failing. + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + candidate, err = e.checkRemoteLeaseOnce(context.Background()) + require.Error(t, err, "reads are still failing") + assert.True(t, candidate, + "an unreachable Active whose newest known renewTime has aged out must become a candidate — "+ + "this is the case that was previously undetectable") +} + +// TestReadFailure_DoesNotRefreshLastGoodRead guards the other half of the +// retention rule: a failed read must not advance any freshness marker. +func TestReadFailure_DoesNotRefreshLastGoodRead(t *testing.T) { + fresh := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now()) + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t, fresh), Options{Mode: ModeStandby, Log: testLog()}) + + _, err := e.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + armedAt := e.lastGoodRead + require.False(t, armedAt.IsZero()) + + e.remoteClient = fakeClient(t) + _, err = e.checkRemoteLeaseOnce(context.Background()) + require.Error(t, err) + assert.Equal(t, armedAt, e.lastGoodRead, "a failed read must not count as a good read") +} + +// TestSuccessfulRead_ReplacesCachedLease covers the recovery direction: an +// Active that comes back must clear the candidacy, not leave a stale verdict +// latched. +func TestSuccessfulRead_ReplacesCachedLease(t *testing.T) { + stale := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t, stale), Options{Mode: ModeStandby, Log: testLog()}) + + candidate, err := e.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + require.True(t, candidate, "an old renewTime reads as stale") + + // The Active recovers and renews. + e.remoteClient = fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + + candidate, err = e.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + assert.False(t, candidate, "a recovered Active must clear candidacy on the next successful read") +} + +// TestCheckRemoteLeaseOnce_BoundsTheRead is a regression test for a bug found +// by live-testing an Active whose API server was shut down: a single poll +// blocked for ~12 seconds, and because the watch loop calls this synchronously, +// no staleness was evaluated for that whole window. A graceful container +// shutdown eventually breaks the connection; a powered-off node or a +// dropped-packet partition does not, and the read would hang for the transport +// timeout — minutes — blowing the failover budget by waiting rather than by +// deciding. +func TestCheckRemoteLeaseOnce_BoundsTheRead(t *testing.T) { + blocked := make(chan struct{}) + defer close(blocked) + + hanging := fake.NewClientBuilder().WithScheme(testScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-blocked: + return nil + } + }, + }).Build() + + e := NewClusterLeaderElector(fakeClient(t), hanging, Options{ + Mode: ModeStandby, + PromotionDialTimeout: 50 * time.Millisecond, + Log: testLog(), + }) + + done := make(chan struct{}) + go func() { + defer close(done) + _, err := e.checkRemoteLeaseOnce(context.Background()) + assert.Error(t, err, "a timed-out read must surface as an error, not as a fresh lease") + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("checkRemoteLeaseOnce did not bound its read — an unresponsive API server would " + + "block the watch loop and stall staleness evaluation entirely") + } +} + +// TestCheckRemoteLeaseOnce_TimedOutReadStillAgesTheCache: the bound must not +// change the verdict logic. A read that times out is a failed read, so the +// retained view is kept and continues to age exactly as it would if the read +// had failed outright. +func TestCheckRemoteLeaseOnce_TimedOutReadStillAgesTheCache(t *testing.T) { + blocked := make(chan struct{}) + defer close(blocked) + + hanging := fake.NewClientBuilder().WithScheme(testScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + <-ctx.Done() + return ctx.Err() + }, + }).Build() + + e := NewClusterLeaderElector(fakeClient(t), hanging, Options{ + Mode: ModeStandby, + PromotionDialTimeout: 20 * time.Millisecond, + Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + candidate, err := e.checkRemoteLeaseOnce(context.Background()) + require.Error(t, err) + assert.True(t, candidate, + "a timed-out read is still just a failed read: the retained view ages and the hub becomes a candidate") + assert.NotNil(t, e.lastSeenLease, "and the retained view must survive the timeout") +} diff --git a/pkg/ha/lease.go b/pkg/ha/lease.go new file mode 100644 index 000000000..dc7064440 --- /dev/null +++ b/pkg/ha/lease.go @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// acquireOrRenewLease ensures a Lease named name/namespace exists, is held by +// identity, and has its renewTime stamped to now. It creates the Lease if it is +// missing and increments leaseTransitions whenever leadership changes hands. +// It is used by the Active's renewal loop (StartLeaseRenewal → renewOnce). +func acquireOrRenewLease(ctx context.Context, c client.Client, name, namespace, identity string, leaseDuration time.Duration) (*coordinationv1.Lease, error) { + now := metav1.NewMicroTime(time.Now()) + // Round up to whole seconds and enforce a minimum of 1s. LeaseDurationSeconds + // is an int32 count of seconds, so a sub-second duration must not truncate to + // 0 (an invalid lease duration that also skews staleness checks). + durationSeconds := int32((leaseDuration + time.Second - 1) / time.Second) + if durationSeconds < 1 { + durationSeconds = 1 + } + + lease := &coordinationv1.Lease{} + err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, lease) + if apierrors.IsNotFound(err) { + transitions := int32(0) + lease = &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &identity, + LeaseDurationSeconds: &durationSeconds, + AcquireTime: &now, + RenewTime: &now, + LeaseTransitions: &transitions, + }, + } + if err := c.Create(ctx, lease); err != nil { + return nil, err + } + return lease, nil + } + if err != nil { + return nil, err + } + + // The Lease exists. If it was held by someone else (or no one), record a + // transition and a fresh acquireTime; otherwise simply renew it. + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != identity { + transitions := int32(1) + if lease.Spec.LeaseTransitions != nil { + transitions = *lease.Spec.LeaseTransitions + 1 + } + lease.Spec.HolderIdentity = &identity + lease.Spec.AcquireTime = &now + lease.Spec.LeaseTransitions = &transitions + } + lease.Spec.LeaseDurationSeconds = &durationSeconds + lease.Spec.RenewTime = &now + if err := c.Update(ctx, lease); err != nil { + return nil, err + } + return lease, nil +} + +// getLease fetches a Lease. The Standby uses this to read the Active's Lease over +// the remote client (WatchRemoteLease → checkRemoteLeaseOnce). +func getLease(ctx context.Context, c client.Client, name, namespace string) (*coordinationv1.Lease, error) { + lease := &coordinationv1.Lease{} + if err := c.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, lease); err != nil { + return nil, err + } + return lease, nil +} + +// isLeaseStale reports whether the lease's renewTime is older than +// leaseDuration + padding relative to now. A nil lease or a missing renewTime is +// treated as stale: the holder has never checked in. +func isLeaseStale(lease *coordinationv1.Lease, padding time.Duration, now time.Time) bool { + if lease == nil || lease.Spec.RenewTime == nil { + return true + } + leaseDuration := time.Duration(0) + if lease.Spec.LeaseDurationSeconds != nil { + leaseDuration = time.Duration(*lease.Spec.LeaseDurationSeconds) * time.Second + } + deadline := lease.Spec.RenewTime.Time.Add(leaseDuration + padding) + return now.After(deadline) +} diff --git a/pkg/ha/lease_test.go b/pkg/ha/lease_test.go new file mode 100644 index 000000000..5a01fb47e --- /dev/null +++ b/pkg/ha/lease_test.go @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +// ---- shared test helpers, used across the ha package tests ---- + +func testScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + return scheme +} + +func fakeClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + return fake.NewClientBuilder().WithScheme(testScheme(t)).WithObjects(objs...).Build() +} + +// failingWriteClient returns a client whose Create and Update always error, +// simulating an unreachable API server (used to exercise natural fencing). +func failingWriteClient(t *testing.T) client.Client { + t.Helper() + return fake.NewClientBuilder().WithScheme(testScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + return fmt.Errorf("simulated API server down") + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + return fmt.Errorf("simulated API server down") + }, + }).Build() +} + +func testLog() *zap.SugaredLogger { + return zap.NewNop().Sugar() +} + +func int32Ptr(i int32) *int32 { return &i } + +func microTimePtr(tm time.Time) *metav1.MicroTime { + m := metav1.NewMicroTime(tm) + return &m +} + +func newLease(name, namespace, holder string, renew time.Time) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Spec: coordinationv1.LeaseSpec{ + HolderIdentity: &holder, + LeaseDurationSeconds: int32Ptr(15), + RenewTime: microTimePtr(renew), + }, + } +} + +// ---- tests ---- + +func TestIsLeaseStale(t *testing.T) { + now := time.Now() + padding := 5 * time.Second + + fresh := newLease("l", "ns", "hub-a", now.Add(-2*time.Second)) + stale := newLease("l", "ns", "hub-a", now.Add(-60*time.Second)) + noRenew := &coordinationv1.Lease{Spec: coordinationv1.LeaseSpec{LeaseDurationSeconds: int32Ptr(15)}} + + assert.False(t, isLeaseStale(fresh, padding, now), "fresh lease should not be stale") + assert.True(t, isLeaseStale(stale, padding, now), "old renewTime should be stale") + assert.True(t, isLeaseStale(noRenew, padding, now), "missing renewTime should be stale") + assert.True(t, isLeaseStale(nil, padding, now), "nil lease should be stale") +} + +func TestAcquireOrRenewLease_CreatesThenRenews(t *testing.T) { + ctx := context.Background() + c := fakeClient(t) + + lease, err := acquireOrRenewLease(ctx, c, "l", "ns", "hub-a", 15*time.Second) + require.NoError(t, err) + require.NotNil(t, lease.Spec.HolderIdentity) + assert.Equal(t, "hub-a", *lease.Spec.HolderIdentity) + require.NotNil(t, lease.Spec.RenewTime) + require.NotNil(t, lease.Spec.LeaseTransitions) + assert.Equal(t, int32(0), *lease.Spec.LeaseTransitions) + firstRenew := lease.Spec.RenewTime.Time + + time.Sleep(2 * time.Millisecond) + lease2, err := acquireOrRenewLease(ctx, c, "l", "ns", "hub-a", 15*time.Second) + require.NoError(t, err) + assert.Equal(t, "hub-a", *lease2.Spec.HolderIdentity) + assert.False(t, lease2.Spec.RenewTime.Time.Before(firstRenew), "renewTime should not move backwards") + assert.Equal(t, int32(0), *lease2.Spec.LeaseTransitions, "same holder should not bump transitions") +} + +func TestAcquireOrRenewLease_TakeoverBumpsTransitions(t *testing.T) { + ctx := context.Background() + existing := newLease("l", "ns", "hub-a", time.Now().Add(-time.Hour)) + existing.Spec.LeaseTransitions = int32Ptr(0) + c := fakeClient(t, existing) + + lease, err := acquireOrRenewLease(ctx, c, "l", "ns", "hub-b", 15*time.Second) + require.NoError(t, err) + assert.Equal(t, "hub-b", *lease.Spec.HolderIdentity) + require.NotNil(t, lease.Spec.LeaseTransitions) + assert.Equal(t, int32(1), *lease.Spec.LeaseTransitions, "takeover should increment transitions") +} + +func TestAcquireOrRenewLease_SubSecondDurationClampsToOne(t *testing.T) { + ctx := context.Background() + c := fakeClient(t) + + lease, err := acquireOrRenewLease(ctx, c, "l", "ns", "hub-a", 500*time.Millisecond) + require.NoError(t, err) + require.NotNil(t, lease.Spec.LeaseDurationSeconds) + assert.Equal(t, int32(1), *lease.Spec.LeaseDurationSeconds, + "a sub-second duration must clamp to 1s, not truncate to 0") +} + +func TestGetLease_NotFoundReturnsError(t *testing.T) { + c := fakeClient(t) + _, err := getLease(context.Background(), c, "missing", "ns") + assert.Error(t, err, "a missing lease must surface as an error, not a nil/zero value") +} diff --git a/pkg/ha/lifecycle_events.go b/pkg/ha/lifecycle_events.go new file mode 100644 index 000000000..cab954043 --- /dev/null +++ b/pkg/ha/lifecycle_events.go @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "time" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + coordinationv1 "k8s.io/api/coordination/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" +) + +// The HA lifecycle Events of issue #298, alongside PromotedToActive which lives +// in promotion_event.go because only promotion holds the object it attaches to. +// +// All four hang off the leader-election Lease, for the reason set out in +// PromotedToActiveEmitter: the recorder derives an Event's namespace from its +// involved object, so a namespaced object is required to land the Event beside +// the controller that emitted it, and the Lease is the one object that both +// exists for this purpose and *is* the leadership record. That makes +// `kubectl -n kubeslice-controller get events` a single place to read the whole +// HA history of a hub. +// +// leaseReference builds that reference by name rather than reading the Lease, +// which is what lets BecameStandby work at all: a Standby has no local Lease +// until the day it promotes, and an Event whose involvedObject names an object +// that does not exist yet is well-formed — the reference carries kind, namespace +// and name, and only the UID is empty. + +// emitLifecycleEvent records one HA lifecycle Event against this hub's Lease. +// +// Failures are logged and swallowed. Every caller is on a path where the Event +// is a report of something that has already happened — leadership already lost, +// promotion already abandoned — so failing the caller because the report failed +// would turn an observability gap into an outage. +// +// recorder.RecordEvent is called directly, never util.RecordEvent, for the +// reason promotion_event.go documents at length: that helper starts with +// util.CtxLogger(ctx), which nil-panics on any context that has not been through +// PrepareKubeSliceControllersRequestContext, and every context here comes from +// main.go's signal handler rather than from a reconciler. +func (e *ClusterLeaderElector) emitLifecycleEvent(ctx context.Context, name events.EventName) { + if e.eventRecorder == nil { + return + } + if err := e.eventRecorder.RecordEvent(ctx, &events.Event{ + Object: leaseReference(e.leaseName, e.leaseNS), + ReportingInstance: util.InstanceController, + Name: name, + }); err != nil { + e.log.Warnw("failed to record HA lifecycle event", + "event", name, "lease", e.leaseName, "namespace", e.leaseNS, "error", err) + } +} + +// EmitStartupModeEvent records BecameActive or BecameStandby for the mode this +// hub started in. Standalone records nothing: it is the pre-HA behaviour, and an +// Event announcing that HA is switched off would appear on every non-HA +// deployment in existence. +// +// Exported and called from main.go rather than fired inside the constructor, so +// that construction stays free of API-server writes — every existing test builds +// an elector, and none of them should start recording Events by doing so. +func (e *ClusterLeaderElector) EmitStartupModeEvent(ctx context.Context) { + switch e.Mode() { + case ModeActive: + e.emitLifecycleEvent(ctx, ossEvents.EventHABecameActive) + case ModeStandby: + e.emitLifecycleEvent(ctx, ossEvents.EventHABecameStandby) + } +} + +// abortPromotion records a refusal to promote, in both surfaces at once: the +// counter labelled with which guard fired, and one PromotionAborted Event. +// +// Single helper rather than a metric increment at each site, because the two +// must not drift — a new abort path that increments the counter and forgets the +// Event (or the reverse) is the kind of gap nobody notices until the one +// failover that needed it. The reason lives only on the metric label and in the +// logs: EventSchema fixes an Event's Message at generation time, so a +// per-reason Event would mean six schema entries for one condition. +func (e *ClusterLeaderElector) abortPromotion(ctx context.Context, reason string) { + haPromotionsAbortedTotal.WithLabelValues(reason).Inc() + e.emitLifecycleEvent(ctx, ossEvents.EventHAPromotionAborted) +} + +// leaseReference is a Lease valued only for its name and namespace — enough for +// an Event's involvedObject, and never read or written as an object. +func leaseReference(name, namespace string) *coordinationv1.Lease { + return &coordinationv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +// remoteLeaseAge reports how long ago the given Lease was renewed. The bool is +// false when there is nothing to measure — no Lease read yet, or a Lease with no +// renewTime — which callers must treat as "do not publish", not as zero: an age +// of zero means "renewed just now", the exact opposite of "unknown". +// +// A negative age is clamped to zero rather than reported. It means the Active's +// clock is ahead of this hub's, and a gauge that dips below zero during skew +// would make an age-based alert flap for a reason unrelated to the Active's +// health. The staleness verdict has its own tolerance for this in padding. +func remoteLeaseAge(lease *coordinationv1.Lease, now time.Time) (time.Duration, bool) { + if lease == nil || lease.Spec.RenewTime == nil { + return 0, false + } + age := now.Sub(lease.Spec.RenewTime.Time) + if age < 0 { + return 0, true + } + return age, true +} diff --git a/pkg/ha/lifecycle_events_test.go b/pkg/ha/lifecycle_events_test.go new file mode 100644 index 000000000..daf0547b6 --- /dev/null +++ b/pkg/ha/lifecycle_events_test.go @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "testing" + "time" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" +) + +// eventsWithReason lists the events on c carrying the given reason. Reason +// rather than event name, because reason is what an operator filters on with +// `kubectl get events --field-selector reason=...` and therefore what the +// runbook documents. +func eventsWithReason(t *testing.T, c client.Client, reason string) []corev1.Event { + t.Helper() + list := &corev1.EventList{} + require.NoError(t, c.List(context.Background(), list)) + var out []corev1.Event + for _, ev := range list.Items { + if ev.Reason == reason { + out = append(out, ev) + } + } + return out +} + +// standbyWithRecorder builds a Standby wired to a recorder backed by c. +func standbyWithRecorder(t *testing.T, c client.Client, mode HAMode) *ClusterLeaderElector { + t.Helper() + return NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: mode, + Identity: "hub-b", + EventRecorder: testEventRecorder(t, c, ossEvents.EventsMap), + Log: testLog(), + }) +} + +// TestHALifecycleEvents_RegisteredInGeneratedMap pins the generate-events step +// this feature depends on. RecordEvent hard-fails for a name that is not in +// EventsMap, so an entry added to config/events/controller.yaml without +// re-running `make generate-events` produces code that compiles, emits nothing, +// and logs a warning nobody reads. +func TestHALifecycleEvents_RegisteredInGeneratedMap(t *testing.T) { + for _, name := range []events.EventName{ + ossEvents.EventHABecameActive, + ossEvents.EventHABecameStandby, + ossEvents.EventHALeadershipLost, + ossEvents.EventHAPromotionAborted, + } { + require.Contains(t, ossEvents.EventsMap, name, + "%s must be in the generated EventsMap — re-run `make generate-events`", name) + } + + // The reasons are the operator-facing contract from issue #298's table, and + // they deliberately differ from the internal event names. + assert.Equal(t, "BecameActive", ossEvents.EventsMap[ossEvents.EventHABecameActive].Reason) + assert.Equal(t, "BecameStandby", ossEvents.EventsMap[ossEvents.EventHABecameStandby].Reason) + assert.Equal(t, "LeadershipLost", ossEvents.EventsMap[ossEvents.EventHALeadershipLost].Reason) + assert.Equal(t, "PromotionAborted", ossEvents.EventsMap[ossEvents.EventHAPromotionAborted].Reason) + + // Severity matters: these three are the ones an operator should see without + // going looking for them. + assert.Equal(t, events.EventTypeWarning, ossEvents.EventsMap[ossEvents.EventHALeadershipLost].Type) + assert.Equal(t, events.EventTypeWarning, ossEvents.EventsMap[ossEvents.EventHAPromotionAborted].Type) + assert.Equal(t, events.EventTypeNormal, ossEvents.EventsMap[ossEvents.EventHABecameActive].Type) +} + +func TestEmitStartupModeEvent_RecordsTheModeThisHubStartedIn(t *testing.T) { + ctx := context.Background() + + standbyEvents := fakeClient(t) + standbyWithRecorder(t, standbyEvents, ModeStandby).EmitStartupModeEvent(ctx) + assert.Len(t, eventsWithReason(t, standbyEvents, "BecameStandby"), 1) + assert.Empty(t, eventsWithReason(t, standbyEvents, "BecameActive")) + + activeEvents := fakeClient(t) + standbyWithRecorder(t, activeEvents, ModeActive).EmitStartupModeEvent(ctx) + assert.Len(t, eventsWithReason(t, activeEvents, "BecameActive"), 1) + assert.Empty(t, eventsWithReason(t, activeEvents, "BecameStandby")) +} + +// TestEmitStartupModeEvent_StandaloneIsSilent is the no-regression guarantee. +// Standalone is the default mode and the pre-HA behaviour; an Event announcing +// that HA is switched off would appear on every non-HA deployment there is. +func TestEmitStartupModeEvent_StandaloneIsSilent(t *testing.T) { + c := fakeClient(t) + standbyWithRecorder(t, c, ModeStandalone).EmitStartupModeEvent(context.Background()) + + list := &corev1.EventList{} + require.NoError(t, c.List(context.Background(), list)) + assert.Empty(t, list.Items, "standalone mode must record no HA lifecycle events at all") +} + +// TestEmitLifecycleEvent_AttachesToTheLeaseInTheControllerNamespace pins where +// these land. The recorder derives an Event's namespace from its involved +// object, so naming the Lease is what puts the Event beside the controller that +// emitted it — and NOT in kubeslice-system, which is a worker namespace that +// does not exist on a hub at all. +func TestEmitLifecycleEvent_AttachesToTheLeaseInTheControllerNamespace(t *testing.T) { + c := fakeClient(t) + e := standbyWithRecorder(t, c, ModeStandby) + e.EmitStartupModeEvent(context.Background()) + + got := eventsWithReason(t, c, "BecameStandby") + require.Len(t, got, 1) + assert.Equal(t, DefaultLeaseNamespace, got[0].Namespace) + assert.Equal(t, DefaultLeaseName, got[0].InvolvedObject.Name) + assert.Equal(t, "Lease", got[0].InvolvedObject.Kind) +} + +// TestEmitLifecycleEvent_WorksBeforeTheLeaseExists is why leaseReference builds +// a reference by name instead of reading the object. A Standby has no local +// Lease until the day it promotes, so requiring one would mean BecameStandby — +// the one mode event a Standby can emit — could never be recorded. +func TestEmitLifecycleEvent_WorksBeforeTheLeaseExists(t *testing.T) { + c := fakeClient(t) // no Lease anywhere + e := standbyWithRecorder(t, c, ModeStandby) + + e.EmitStartupModeEvent(context.Background()) + + assert.Len(t, eventsWithReason(t, c, "BecameStandby"), 1, + "the event must record against a Lease that does not exist yet") +} + +func TestEmitLifecycleEvent_NilRecorderIsANoop(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + require.Nil(t, e.eventRecorder) + + // Must not panic, and must leave every other effect intact. + assert.NotPanics(t, func() { + e.EmitStartupModeEvent(context.Background()) + e.emitLifecycleEvent(context.Background(), ossEvents.EventHALeadershipLost) + }) +} + +// TestEmitLifecycleEvent_RecorderFailureIsSwallowed keeps an observability gap +// from becoming an outage. Every caller is reporting something that has already +// happened, so a failed Event write must not fail the caller. +func TestEmitLifecycleEvent_RecorderFailureIsSwallowed(t *testing.T) { + // An EventsMap without the HA entries makes RecordEvent return an error. + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + EventRecorder: testEventRecorder(t, fakeClient(t), map[events.EventName]*events.EventSchema{}), + Log: testLog(), + }) + + assert.NotPanics(t, func() { e.EmitStartupModeEvent(context.Background()) }) +} + +func TestAbortPromotion_CountsTheReasonAndEmitsOneEvent(t *testing.T) { + haPromotionsAbortedTotal.Reset() + c := fakeClient(t) + e := standbyWithRecorder(t, c, ModeStandby) + + e.abortPromotion(context.Background(), abortSelfUnhealthy) + + assert.Equal(t, float64(1), testutil.ToFloat64(haPromotionsAbortedTotal.WithLabelValues(abortSelfUnhealthy))) + got := eventsWithReason(t, c, "PromotionAborted") + require.Len(t, got, 1, "a refusal to promote must be visible as an Event, not only as a metric") + assert.Equal(t, string(corev1.EventTypeWarning), got[0].Type) +} + +// TestGuardRefusal_EmitsPromotionAborted checks the wiring end to end: a live +// Active must produce both the metric and the Event through the real guard path, +// not just through a direct call to the helper. +func TestGuardRefusal_EmitsPromotionAborted(t *testing.T) { + ctx := context.Background() + haPromotionsAbortedTotal.Reset() + + remote := fakeClient(t) + require.NoError(t, remote.Create(ctx, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now()))) + + eventsClient := fakeClient(t) + e := NewClusterLeaderElector(fakeClient(t), remote, Options{ + Mode: ModeStandby, + Identity: "hub-b", + EventRecorder: testEventRecorder(t, eventsClient, ossEvents.EventsMap), + Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + promoted, err := e.promote(ctx) + require.NoError(t, err) + require.False(t, promoted) + + assert.Len(t, eventsWithReason(t, eventsClient, "PromotionAborted"), 1) + assert.Equal(t, float64(1), testutil.ToFloat64(haPromotionsAbortedTotal.WithLabelValues(abortLeaseLive))) +} + +// TestRenewOnce_EmitsLeadershipLostOnlyPastTheRenewDeadline is the distinction +// issue #298's table draws: the Warning belongs to an Active that has actually +// given up leadership, not to every transient renewal failure. A hub still +// inside renewDeadline is expected to keep going quietly. +func TestRenewOnce_EmitsLeadershipLostOnlyPastTheRenewDeadline(t *testing.T) { + ctx := context.Background() + + // Inside the deadline: leadership retained, no Event. + quiet := fakeClient(t) + within := NewClusterLeaderElector(failingWriteClient(t), nil, Options{ + Mode: ModeActive, + Identity: "hub-a", + RenewDeadline: time.Hour, + EventRecorder: testEventRecorder(t, quiet, ossEvents.EventsMap), + Log: testLog(), + }) + within.setLeader(true) + within.lastRenew = time.Now() + require.Error(t, within.renewOnce(ctx)) + assert.True(t, within.IsLeader(), "a failure inside the renew deadline keeps leadership") + assert.Empty(t, eventsWithReason(t, quiet, "LeadershipLost")) + + // Past the deadline: leadership released, exactly one Event. + loud := fakeClient(t) + past := NewClusterLeaderElector(failingWriteClient(t), nil, Options{ + Mode: ModeActive, + Identity: "hub-a", + RenewDeadline: 10 * time.Millisecond, + EventRecorder: testEventRecorder(t, loud, ossEvents.EventsMap), + Log: testLog(), + }) + past.setLeader(true) + past.lastRenew = time.Now().Add(-time.Hour) + require.Error(t, past.renewOnce(ctx)) + assert.False(t, past.IsLeader(), "a failure past the renew deadline must release leadership") + assert.Len(t, eventsWithReason(t, loud, "LeadershipLost"), 1) +} + +// TestStartLeaseRenewal_ShutdownDoesNotEmitLeadershipLost is why the Event is +// emitted in renewOnce rather than in setLeader. Graceful shutdown also drops +// leadership, and a Warning Event on every rolling restart is noise — besides +// racing the pod's own termination for the write. +func TestStartLeaseRenewal_ShutdownDoesNotEmitLeadershipLost(t *testing.T) { + c := fakeClient(t) + e := NewClusterLeaderElector(fakeClient(t), nil, Options{ + Mode: ModeActive, + Identity: "hub-a", + RetryPeriod: time.Hour, // never ticks; only ctx cancellation ends the loop + EventRecorder: testEventRecorder(t, c, ossEvents.EventsMap), + Log: testLog(), + }) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.NoError(t, e.StartLeaseRenewal(ctx)) + + assert.False(t, e.IsLeader()) + assert.Empty(t, eventsWithReason(t, c, "LeadershipLost"), + "a graceful shutdown must not report lost leadership as a warning") +} diff --git a/pkg/ha/metrics.go b/pkg/ha/metrics.go new file mode 100644 index 000000000..43ed64894 --- /dev/null +++ b/pkg/ha/metrics.go @@ -0,0 +1,419 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// HA's metrics, registered on controller-runtime's registry (issue #298). +// +// The registry choice is the one decision here that changes whether any of this +// is observable at all, so it is stated rather than left to be discovered. +// controller-runtime's metrics server serves ctrlmetrics.Registry and nothing +// else (pkg/metrics/server/server.go builds its handler with +// promhttp.HandlerFor(metrics.Registry, ...)), which is the endpoint +// --metrics-bind-address exposes and the one kube-rbac-proxy fronts in +// config/default/manager_auth_proxy_patch.yaml. These metrics were previously +// registered with prometheus.MustRegister, i.e. onto the client library's +// DEFAULT registry — which this repo does serve, but only from +// metrics.StartMetricsCollector's own ListenAndServe on service.MetricPort +// (18080), a port that appears in no manifest in config/: no containerPort, no +// Service, and nothing for the kube-rbac-proxy to authenticate. So every HA +// metric was being collected and then published where nothing could scrape it. +// +// metrics/prometheus.go looks like a precedent for the default registry but is +// not one: KubeSliceEventsCounter is created through a factory built on +// ctrlmetrics.Registry (prometheus.go:38) and only additionally registered on +// the default one, so it reaches the standard endpoint by the first path. The +// monitoring framework's own default labels are slice-specific and do not apply +// to a cross-cluster mirror, which is why these stay hand-rolled rather than +// going through mfm.NewMetricsFactory. +// +// **Why the role-scoped gauges carry a `mode` label.** Several of these describe +// a role rather than the process: a Standby has no meaningful +// ha_lease_last_renew_time_seconds any more than an Active has a meaningful +// ha_remote_lease_age_seconds, and nothing has a meaningful +// ha_last_promotion_timestamp_seconds until it has actually promoted. Those must +// be ABSENT where they do not apply, not zero — a zeroed timestamp gauge reads +// as 1970, so `time() - metric` returns decades and any alert built on it fires +// permanently. +// +// Simply not calling Set() does NOT achieve that, which is the trap this layout +// exists to avoid. A plain registered Gauge always collects, reporting 0 until +// something sets it; only a *Vec with no children collects nothing at all. So +// every role-scoped gauge here is a GaugeVec labelled `mode`, and its child is +// created only by the role it belongs to. Verified on a live pair: an Active +// publishes no ha_armed and no ha_remote_lease_age_seconds, a Standby publishes +// no ha_lease_last_renew_time_seconds, and a hub that has never promoted +// publishes no ha_last_promotion_timestamp_seconds. +// +// ha_leader_status is deliberately NOT scoped this way. It is a plain Gauge that +// both roles publish, because 0 is its alertable value and +// `sum(ha_leader_status) != 1` — no Active, or two — has to be expressible across +// the pair. ha_sync_queue_depth stays plain for the mundane reason that 0 is +// truthful on an Active: there is no backlog because there is no queue. +var ( + // haLeaderStatus is the write fence, exported. 1 means this instance holds + // leadership and its reconcilers are writing; 0 means they are not. + // + // It tracks the durable isLeader flag, so it reads 0 for an Active that has + // lost its Lease but is still running — which is the point, that being the + // state worth paging on. It is deliberately not IsLeader(), which also + // reports false for the duration of a promotion; that window is what + // ha_promotion_duration_seconds measures. + haLeaderStatus = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_leader_status", + Help: "1 if this controller instance currently holds HA leadership (Active), 0 if it does not (Standby).", + }) + + // haLeaseLastRenewTime is the Unix timestamp of the last successful renewal + // of this hub's own Lease. Active only. Alert on age, not on the value: + // `time() - kubeslice_controller_ha_lease_last_renew_time_seconds` crossing + // renewDeadline means leadership is about to be released. + haLeaseLastRenewTime = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_lease_last_renew_time_seconds", + Help: "Unix timestamp of the last successful renewal of this hub's own HA Lease (Active only).", + }, []string{"mode"}) + + // haLeaseRenewErrorsTotal counts failed renewal attempts. These are the + // near-misses that precede a self-demotion: renewOnce keeps leadership while + // it is still inside renewDeadline, so a hub can be failing every renewal + // for seconds before LeadershipLost fires and this is the only signal in + // that window. + haLeaseRenewErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_lease_renew_errors_total", + Help: "Count of failed attempts to renew this hub's own HA Lease.", + }) + + // haSyncLagSeconds observes, per kind and operation, how far behind the + // mirror is: time.Now() minus the source object's CreationTimestamp for + // creates, or minus the time the object was first enqueued for + // update/delete (the more useful number to alert on once a retry has + // backed off a few times — it reflects total time since the triggering + // change, not just the last dequeue). + // + // Buckets are the ones issue #298 specifies. They are wider and coarser than + // prometheus.DefBuckets, which was the previous value and the wrong shape + // here: DefBuckets spends five of its eleven buckets below 100ms, a range a + // cross-cluster mirror never operates in, and stops at 10s, well short of + // the multi-second-to-minutes lag that actually matters. + haSyncLagSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "kubeslice_controller", + Name: "ha_sync_lag_seconds", + Help: "Time between a change on the Active hub and it being reflected on the Standby.", + Buckets: syncLagBuckets, + }, []string{"kind", "operation"}) + + // haSyncErrorsTotal counts mirror failures. The syncer keeps running and + // retries via its workqueue on every increment — this metric never + // indicates a crash, only a retry in progress. + haSyncErrorsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_sync_errors_total", + Help: "Count of mirror sync failures, by kind and operation.", + }, []string{"kind", "operation"}) + + // haSyncQueueDepth is the mirror workqueue's length. + // + // It answers a question ha_sync_lag_seconds structurally cannot: lag is only + // observed for items that finished, so a syncer wedged behind a growing + // backlog reports healthy lag from the few items still completing while + // falling further behind. Depth is what distinguishes "keeping up" from + // "keeping up with a fraction of the work". + haSyncQueueDepth = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_sync_queue_depth", + Help: "Number of keys currently waiting in the mirror workqueue.", + }) + + // haFailoverTotal counts completed promotions. A Standby that takes over + // increments this exactly once, after the write fence has opened. + haFailoverTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_failover_total", + Help: "Count of completed promotions from Standby to Active.", + }) + + // haLastPromotionTimestamp is the Unix timestamp of the last completed + // promotion, for `time() - metric` = "how long have we been running on the + // promoted hub". + // + // Process-local, like every counter here: a restart clears it, and an + // unset gauge means "this process has not promoted", NOT "this hub never + // did". The durable record of a past failover is the PromotedToActive Event + // and the Lease's own holderIdentity/renewTime. + haLastPromotionTimestamp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_last_promotion_timestamp_seconds", + Help: "Unix timestamp of the last promotion this process completed.", + }, []string{"mode"}) + + // haPromotionDurationSeconds is the wall time of the whole promote() + // sequence, labelled by outcome. + // + // The label is load-bearing rather than decorative. An aborted attempt's + // duration is genuinely worth having — a StopMirror that times out spends + // the entire promotionGracePeriod before giving up — but averaged in with + // real promotions it would corrupt the only number anyone actually asks for, + // which is how long a successful failover takes. + haPromotionDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "kubeslice_controller", + Name: "ha_promotion_duration_seconds", + Help: "Wall-clock duration of the promotion sequence, by outcome.", + Buckets: promotionBuckets, + }, []string{"outcome"}) + + // haPromotionStepDurationSeconds breaks that total down by step. + // + // The total says a promotion was slow; only the breakdown says which of the + // bounded steps spent the budget, and they fail for unrelated reasons — a + // mirror that will not stop, a local API server that will not take the + // Lease, a kick with nothing draining its channels yet. Every step measured + // here already brackets itself in a context.WithTimeout, so these are the + // same boundaries promotion already treats as its budget units. + haPromotionStepDurationSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "kubeslice_controller", + Name: "ha_promotion_step_duration_seconds", + Help: "Wall-clock duration of each individual step of the promotion sequence.", + Buckets: promotionBuckets, + }, []string{"step"}) + + // haFailoverDetectionSeconds is how long it took to notice: the Active's + // last observed renewTime to the moment this hub committed to promoting. + // + // This is the number that validates the failover budget empirically. Added + // to ha_promotion_duration_seconds it is the total window in which the + // cluster had no writer, which is the only figure an operator with an SLO + // cares about. By construction it lands near leaseDuration + padding; the + // spread above that is the cost of polling every retryPeriod. + // + // Recorded once per *successful* promotion rather than on every stale tick. + // Sampling it per tick would mean a hub whose guards keep refusing emits an + // ever-growing detection time forever, which describes the refusal rather + // than any detection. + haFailoverDetectionSeconds = prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: "kubeslice_controller", + Name: "ha_failover_detection_seconds", + Help: "Time from the Active hub's last observed Lease renewal to this hub committing to promotion.", + Buckets: detectionBuckets, + }) + + // haPromotionsAbortedTotal counts the times a Standby decided the Active + // looked gone and then refused to promote anyway. Without it every guard is + // invisible in production: a hub that correctly declines to take over looks + // identical to one that never noticed anything. These are the branches worth + // demonstrating, because they are what stops a configuration mistake or a + // local network failure from becoming a split brain. + haPromotionsAbortedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_promotions_aborted_total", + Help: "Count of promotions considered and then refused, by reason.", + }, []string{"reason"}) + + // haRemoteLeaseAgeSeconds is the age of the newest Lease this Standby has + // read from the Active: now minus that Lease's renewTime. Standby only. + // + // The leading indicator, and the one gauge to graph. Everything else in this + // file reports a failover that already happened; this one climbs beforehand, + // so an alert at a fraction of leaseDuration + padding fires while there is + // still time to look. It keeps climbing when reads fail, by design — a + // retained stale Lease ageing against a moving clock is exactly how + // checkRemoteLeaseOnce models "no new evidence of life". + haRemoteLeaseAgeSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_remote_lease_age_seconds", + Help: "Age in seconds of the newest Lease this Standby has read from the Active hub.", + }, []string{"mode"}) + + // haRemoteLeaseReadsTotal counts remote Lease reads by result. + // + // A Standby that has silently lost its read path is otherwise + // indistinguishable from a healthy one from the outside: lastSeenLease is + // deliberately not cleared on a failed read, so the cached view stays + // populated and only the logs know. This is the metric that catches an + // expired credential or a kubeconfig aimed at the wrong cluster — the + // failure that presents as x509 errors looping until someone reads the logs. + haRemoteLeaseReadsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_remote_lease_reads_total", + Help: "Count of reads of the Active hub's Lease from this Standby, by result.", + }, []string{"result"}) + + // haArmed reports whether this Standby has ever successfully read the + // Active's Lease, and is therefore eligible to promote at all. + // + // The arming rule is a safety property — a Standby that has never seen the + // Active alive must never conclude it died — but it has a failure mode with + // no other signal: a hub misconfigured badly enough to never arm will never + // fail over, and looks perfectly healthy until the day it is needed. 0 here + // on a Standby means the HA pair is not actually protecting anything. + haArmed = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_armed", + Help: "1 if this Standby has read the Active hub's Lease at least once and could promote, 0 if not.", + }, []string{"mode"}) + + // haPruneResurrectedTotal counts objects the prune pass re-enqueued because + // they exist on the Active with no mirror on the Standby. + // + // Prune is a backstop for drift the event path missed, so a backstop that + // fires steadily is not reassurance — it is evidence the informer path is + // dropping work. Zero is the healthy value, and nothing today would tell + // you it is not zero. + haPruneResurrectedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_prune_resurrected_total", + Help: "Count of Active-side objects the prune pass re-enqueued because the Standby had no mirror.", + }, []string{"kind"}) + + // haPruneLastRunTimestamp is the Unix timestamp of the last completed prune + // pass. Its absence, or an age far past pruneInterval, means the drift + // backstop is not running — which is silent, since a prune pass that never + // happens produces no errors either. + haPruneLastRunTimestamp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "kubeslice_controller", + Name: "ha_prune_last_run_timestamp_seconds", + Help: "Unix timestamp of the last completed prune pass.", + }, []string{"mode"}) + + // haActivePublishErrorsTotal counts failures to write status.activeController. + // + // Publishing is best-effort everywhere by design: promotion logs and + // continues past a failed publish rather than stranding the cluster with no + // writer, and the periodic loop just retries. That is the right trade and it + // is also why the failure is invisible — while being the exact field + // worker-operator #467 reads to find the new Active. This is the difference + // between "failover worked" and "failover worked but no worker noticed". + haActivePublishErrorsTotal = prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "kubeslice_controller", + Name: "ha_active_publish_errors_total", + Help: "Count of failed attempts to publish status.activeController onto this hub's Cluster CRs.", + }) +) + +// Bucket sets. Named rather than inlined because the promotion ones are shared +// by the total and the per-step histograms, which must stay comparable. +var ( + // syncLagBuckets is issue #298's specified set. + syncLagBuckets = []float64{0.1, 0.5, 1, 2, 5, 10, 30} + + // promotionBuckets reaches 60s deliberately. promote() can spend up to four + // sequential promotionGracePeriod budgets (stop mirror, publish, kick, emit) + // plus two promotionDialTimeout guard dials, so the interesting tail sits + // far above DefBuckets' 10s ceiling — and a promotion pinned in the top + // bucket is precisely the case worth seeing. + promotionBuckets = []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60} + + // detectionBuckets starts at 1s because detection cannot be faster than one + // retryPeriod and is bounded below by leaseDuration + padding; sub-second + // buckets would all be empty. + detectionBuckets = []float64{1, 2, 5, 10, 15, 20, 30, 45, 60, 120} +) + +// Reasons recorded on haPromotionsAbortedTotal. +const ( + // abortSelfUnhealthy: this hub could not reach its own API server, so the + // evidence for the Active being gone is equally consistent with this hub + // being the broken one. + abortSelfUnhealthy = "self_unhealthy" + // abortLeaseLive: the final read found a live Lease — the Active renewed + // between polls, so the staleness verdict was a polling race. + abortLeaseLive = "lease_live" + // abortAlreadyPromoting: a concurrent tick is already running the sequence. + abortAlreadyPromoting = "already_promoting" + // abortMirrorNotStopped: the mirror did not confirm it stopped inside the + // grace period, so proceeding would open the fence on a dual writer. + abortMirrorNotStopped = "mirror_not_stopped" + // abortLeaseAcquireFailed: this hub could not take the Lease on its own + // cluster. + abortLeaseAcquireFailed = "lease_acquire_failed" + // abortNoRemoteClient: promote() was called with no client to the Active, so + // nothing could have established that it was ever alive. + abortNoRemoteClient = "no_remote_client" +) + +// Outcomes recorded on haPromotionDurationSeconds. +const ( + outcomePromoted = "promoted" + outcomeAborted = "aborted" +) + +// Steps recorded on haPromotionStepDurationSeconds. The values match the step +// names used in promote()'s own comments and log lines, so a slow step in a +// dashboard and a slow step in the logs are searchable with the same word. +const ( + stepStopMirror = "stop_mirror" + stepAcquireLease = "acquire_lease" + stepPublishActive = "publish_active_controller" + stepKickReconcilers = "kick_reconcilers" + stepEmitPromoted = "emit_event" +) + +// Results recorded on haRemoteLeaseReadsTotal. +const ( + readResultOK = "ok" + readResultError = "error" +) + +func init() { + ctrlmetrics.Registry.MustRegister( + haLeaderStatus, + haLeaseLastRenewTime, + haLeaseRenewErrorsTotal, + haSyncLagSeconds, + haSyncErrorsTotal, + haSyncQueueDepth, + haFailoverTotal, + haLastPromotionTimestamp, + haPromotionDurationSeconds, + haPromotionStepDurationSeconds, + haFailoverDetectionSeconds, + haPromotionsAbortedTotal, + haRemoteLeaseAgeSeconds, + haRemoteLeaseReadsTotal, + haArmed, + haPruneResurrectedTotal, + haPruneLastRunTimestamp, + haActivePublishErrorsTotal, + ) +} + +// boolGauge maps a boolean onto the 1/0 a Prometheus gauge wants. Written once +// here because five call sites would otherwise each inline the same conditional. +func boolGauge(b bool) float64 { + if b { + return 1 + } + return 0 +} + +// observeStep records the duration of one promotion step. +// +// Called explicitly after each step rather than deferred, because promote()'s +// steps are inline stretches of one function rather than separate calls, and a +// defer would fire at the end of the whole sequence instead of the end of the +// step. Every call site therefore sits immediately after that step's cancel(). +func observeStep(step string, start time.Time) { + haPromotionStepDurationSeconds.WithLabelValues(step).Observe(time.Since(start).Seconds()) +} diff --git a/pkg/ha/metrics_test.go b/pkg/ha/metrics_test.go new file mode 100644 index 000000000..97de790e8 --- /dev/null +++ b/pkg/ha/metrics_test.go @@ -0,0 +1,639 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +// haMetricNames is every metric this package publishes, fully qualified. +// +// Written out by hand rather than derived from the collectors, so that the list +// is an independent statement of the contract: a metric renamed in metrics.go +// without a matching change here fails, which is the point, because the names +// are what dashboards, alerts and the runbook are written against. +var haMetricNames = []string{ + "kubeslice_controller_ha_leader_status", + "kubeslice_controller_ha_lease_last_renew_time_seconds", + "kubeslice_controller_ha_lease_renew_errors_total", + "kubeslice_controller_ha_sync_lag_seconds", + "kubeslice_controller_ha_sync_errors_total", + "kubeslice_controller_ha_sync_queue_depth", + "kubeslice_controller_ha_failover_total", + "kubeslice_controller_ha_failover_detection_seconds", + "kubeslice_controller_ha_promotions_aborted_total", + "kubeslice_controller_ha_last_promotion_timestamp_seconds", + "kubeslice_controller_ha_promotion_duration_seconds", + "kubeslice_controller_ha_promotion_step_duration_seconds", + "kubeslice_controller_ha_remote_lease_age_seconds", + "kubeslice_controller_ha_remote_lease_reads_total", + "kubeslice_controller_ha_armed", + "kubeslice_controller_ha_prune_resurrected_total", + "kubeslice_controller_ha_prune_last_run_timestamp_seconds", + "kubeslice_controller_ha_active_publish_errors_total", +} + +// touchEveryVec gives each labelled metric one child. A *Vec with no children +// collects nothing at all, so without this the registry checks below would pass +// vacuously for exactly the metrics most likely to be mis-registered. +func touchEveryVec() { + haSyncLagSeconds.WithLabelValues("SliceConfig", "create").Observe(0.2) + haSyncErrorsTotal.WithLabelValues("SliceConfig", "update").Add(0) + haPromotionsAbortedTotal.WithLabelValues(abortLeaseLive).Add(0) + haPromotionDurationSeconds.WithLabelValues(outcomePromoted).Observe(0.3) + haPromotionStepDurationSeconds.WithLabelValues(stepStopMirror).Observe(0.1) + haRemoteLeaseReadsTotal.WithLabelValues(readResultOK).Add(0) + haPruneResurrectedTotal.WithLabelValues("SliceConfig").Add(0) + haLeaseLastRenewTime.WithLabelValues(string(ModeActive)).Set(1) + haLastPromotionTimestamp.WithLabelValues(string(ModeActive)).Set(1) + haRemoteLeaseAgeSeconds.WithLabelValues(string(ModeStandby)).Set(1) + haArmed.WithLabelValues(string(ModeStandby)).Set(1) + haPruneLastRunTimestamp.WithLabelValues(string(ModeStandby)).Set(1) +} + +// histogramSampleCount reports how many observations one series of a histogram +// has recorded. testutil.ToFloat64 cannot read histograms, and the observation +// count — not the series count — is what the promotion tests need. +func histogramSampleCount(t *testing.T, c prometheus.Collector, wantLabels map[string]string) uint64 { + t.Helper() + ch := make(chan prometheus.Metric, 128) + c.Collect(ch) + close(ch) + + for m := range ch { + var pb dto.Metric + require.NoError(t, m.Write(&pb)) + if pb.Histogram == nil { + continue + } + got := map[string]string{} + for _, l := range pb.GetLabel() { + got[l.GetName()] = l.GetValue() + } + match := true + for k, v := range wantLabels { + if got[k] != v { + match = false + break + } + } + if match { + return pb.Histogram.GetSampleCount() + } + } + return 0 +} + +// histogramUpperBounds returns the bucket boundaries a histogram was built with, +// read back off a collected sample rather than off the source slice — the point +// being to prove the histogram actually carries them. +func histogramUpperBounds(t *testing.T, c prometheus.Collector) []float64 { + t.Helper() + ch := make(chan prometheus.Metric, 128) + c.Collect(ch) + close(ch) + + for m := range ch { + var pb dto.Metric + require.NoError(t, m.Write(&pb)) + if pb.Histogram == nil { + continue + } + var bounds []float64 + for _, b := range pb.Histogram.GetBucket() { + bounds = append(bounds, b.GetUpperBound()) + } + return bounds + } + return nil +} + +// TestHAMetrics_RegisteredOnControllerRuntimeRegistry is issue #298's first +// acceptance criterion, and it pins the bug it was written against: these +// metrics used to be registered with prometheus.MustRegister, i.e. onto the +// client library's default registry, which controller-runtime's metrics server +// does not serve. They were being collected and published on a port that +// appears in no manifest, so /metrics never carried a single one of them. +func TestHAMetrics_RegisteredOnControllerRuntimeRegistry(t *testing.T) { + touchEveryVec() + + families, err := ctrlmetrics.Registry.Gather() + require.NoError(t, err) + + present := map[string]*dto.MetricFamily{} + for _, f := range families { + present[f.GetName()] = f + } + + for _, name := range haMetricNames { + assert.Contains(t, present, name, + "%s must be registered on ctrlmetrics.Registry — that is the only registry "+ + "controller-runtime's metrics server serves", name) + } +} + +// TestHAMetrics_ServedOverHTTPWithHelpAndType is acceptance criteria #1 and #4 +// taken literally: the metrics must be visible on a /metrics endpoint, with HELP +// and TYPE comments in the output. +// +// The handler here is constructed exactly as controller-runtime's own metrics +// server constructs it — promhttp.HandlerFor over ctrlmetrics.Registry, see +// pkg/metrics/server/server.go — so this exercises the real exposition path +// rather than a re-implementation of it, without needing a manager or a cluster. +func TestHAMetrics_ServedOverHTTPWithHelpAndType(t *testing.T) { + touchEveryVec() + + srv := httptest.NewServer(promhttp.HandlerFor(ctrlmetrics.Registry, promhttp.HandlerOpts{})) + defer srv.Close() + + resp, err := http.Get(srv.URL) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + out := string(body) + + for _, name := range haMetricNames { + assert.Contains(t, out, "# HELP "+name, "%s must appear with a HELP comment", name) + assert.Contains(t, out, "# TYPE "+name, "%s must appear with a TYPE comment", name) + } +} + +// TestHAMetrics_HaveHelpAndType is acceptance criterion #4: HELP and TYPE must +// appear in /metrics output. Both are emitted by the exposition format from the +// collector's own description, so asserting they are non-empty here is the same +// guarantee without needing to stand up an HTTP server. +func TestHAMetrics_HaveHelpAndType(t *testing.T) { + touchEveryVec() + + families, err := ctrlmetrics.Registry.Gather() + require.NoError(t, err) + + checked := 0 + for _, f := range families { + name := f.GetName() + if !contains(haMetricNames, name) { + continue + } + checked++ + assert.NotEmpty(t, f.GetHelp(), "%s must carry a HELP string", name) + assert.NotEqual(t, dto.MetricType(-1), f.GetType(), "%s must carry a TYPE", name) + } + assert.Equal(t, len(haMetricNames), checked, "every HA metric must have been reachable to check") +} + +// TestHAMetrics_LintClean runs Prometheus' own naming linter. It is what catches +// the mistakes a human reviewer reliably misses: a counter without the _total +// suffix, a unit that disagrees with the name, a gauge measuring seconds called +// something else. +func TestHAMetrics_LintClean(t *testing.T) { + touchEveryVec() + + for _, c := range []prometheus.Collector{ + haLeaderStatus, + haLeaseLastRenewTime, + haLeaseRenewErrorsTotal, + haSyncLagSeconds, + haSyncErrorsTotal, + haSyncQueueDepth, + haFailoverTotal, + haFailoverDetectionSeconds, + haPromotionsAbortedTotal, + haLastPromotionTimestamp, + haPromotionDurationSeconds, + haPromotionStepDurationSeconds, + haRemoteLeaseAgeSeconds, + haRemoteLeaseReadsTotal, + haArmed, + haPruneResurrectedTotal, + haPruneLastRunTimestamp, + haActivePublishErrorsTotal, + } { + problems, err := testutil.CollectAndLint(c) + require.NoError(t, err) + assert.Empty(t, problems, "metric naming/lint problems: %+v", problems) + } +} + +func TestHASyncMetrics_RecordAndLabel(t *testing.T) { + haSyncLagSeconds.Reset() + haSyncErrorsTotal.Reset() + + haSyncLagSeconds.WithLabelValues("SliceConfig", "create").Observe(0.5) + haSyncErrorsTotal.WithLabelValues("SliceConfig", "update").Inc() + haSyncErrorsTotal.WithLabelValues("SliceConfig", "update").Inc() + + assert.Equal(t, 1, testutil.CollectAndCount(haSyncLagSeconds)) + assert.Equal(t, float64(2), testutil.ToFloat64(haSyncErrorsTotal.WithLabelValues("SliceConfig", "update"))) +} + +// TestHASyncLagSeconds_UsesTheSpecifiedBuckets pins the buckets to issue #298's +// table. The previous value was prometheus.DefBuckets, which spends five of its +// eleven buckets below 100ms — a range a cross-cluster mirror never operates in +// — and stops at 10s, below the lag worth alerting on. +func TestHASyncLagSeconds_UsesTheSpecifiedBuckets(t *testing.T) { + haSyncLagSeconds.Reset() + haSyncLagSeconds.WithLabelValues("SliceConfig", "create").Observe(0.2) + + assert.Equal(t, []float64{0.1, 0.5, 1, 2, 5, 10, 30}, + histogramUpperBounds(t, haSyncLagSeconds)) +} + +// TestHAPromotionHistograms_ReachPastTenSeconds guards the reason promotion has +// its own bucket set: a promotion can spend four sequential grace periods plus +// two guard dials, so a ceiling of 10s would collapse every slow promotion — +// precisely the ones worth investigating — into +Inf. +func TestHAPromotionHistograms_ReachPastTenSeconds(t *testing.T) { + haPromotionDurationSeconds.Reset() + haPromotionDurationSeconds.WithLabelValues(outcomePromoted).Observe(0.1) + + bounds := histogramUpperBounds(t, haPromotionDurationSeconds) + require.NotEmpty(t, bounds) + assert.Equal(t, float64(60), bounds[len(bounds)-1], + "the top bucket must be well above a single grace period") +} + +func TestHALeaderStatus_TracksLeadershipTransitions(t *testing.T) { + // Standalone is always the leader, and the gauge is published at + // construction so the series exists before any transition happens. + NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: ModeStandalone, Log: testLog()}) + assert.Equal(t, float64(1), testutil.ToFloat64(haLeaderStatus), + "standalone must publish 1 immediately — it is unconditionally the leader") + + e := NewClusterLeaderElector(fakeClient(t), nil, Options{Mode: ModeStandby, Identity: "hub-b", Log: testLog()}) + assert.Equal(t, float64(0), testutil.ToFloat64(haLeaderStatus), + "a standby must publish 0 at construction, not leave the series absent") + + e.setLeader(true) + assert.Equal(t, float64(1), testutil.ToFloat64(haLeaderStatus)) + e.setLeader(false) + assert.Equal(t, float64(0), testutil.ToFloat64(haLeaderStatus)) +} + +// TestRoleScopedGauges_AbsentOnTheWrongRole is the regression test for a defect +// that only a live pair exposed. These gauges were plain prometheus.Gauge and +// were simply never Set() on the role they do not describe — which does not make +// them absent. **A registered plain Gauge always collects, reporting 0.** So a +// real Active published `ha_armed 0` (making "unarmed Standby" alerts fire on +// every Active) and `ha_prune_last_run_timestamp_seconds 0` (making the +// "backstop is not running" alert fire forever), and any hub that had not +// promoted published `ha_last_promotion_timestamp_seconds 0`, i.e. 1970. +// +// A *Vec with no children is what actually collects nothing, so each of these is +// now labelled by `mode` and only its own role creates the child. Asserting on +// the gathered output rather than on values, because the whole property under +// test is a series NOT being there. +func TestRoleScopedGauges_AbsentOnTheWrongRole(t *testing.T) { + for _, v := range []*prometheus.GaugeVec{ + haArmed, haRemoteLeaseAgeSeconds, haLeaseLastRenewTime, + haLastPromotionTimestamp, haPruneLastRunTimestamp, + } { + v.Reset() + } + + // An Active that renews: publishes its renew time, and nothing Standby-scoped. + active := NewClusterLeaderElector(fakeClient(t), nil, Options{ + Mode: ModeActive, Identity: "hub-a", Log: testLog(), + }) + require.NoError(t, active.renewOnce(context.Background())) + + assert.Equal(t, 1, testutil.CollectAndCount(haLeaseLastRenewTime), + "an Active must publish its own lease renew time") + assert.Equal(t, 0, testutil.CollectAndCount(haArmed), + "an Active must publish NO ha_armed series — it has no remote hub to be armed against") + assert.Equal(t, 0, testutil.CollectAndCount(haRemoteLeaseAgeSeconds), + "an Active must publish NO remote lease age") + assert.Equal(t, 0, testutil.CollectAndCount(haPruneLastRunTimestamp), + "an Active runs no prune pass and must publish no timestamp for it") + assert.Equal(t, 0, testutil.CollectAndCount(haLastPromotionTimestamp), + "a hub that has not promoted must publish no promotion timestamp, not a zero one") + + // A Standby: the mirror image. + haLeaseLastRenewTime.Reset() + standby := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + _, _ = standby.checkRemoteLeaseOnce(context.Background()) + + assert.Equal(t, 1, testutil.CollectAndCount(haArmed), + "a Standby must publish ha_armed so that 0 is alertable") + assert.Equal(t, 0, testutil.CollectAndCount(haLeaseLastRenewTime), + "a Standby holds no lease of its own and must publish no renew time") +} + +// TestPromote_DropsStandbyScopedSeries covers the other half: a promoted hub has +// stopped watching a remote, so a frozen remote-lease age left behind would look +// exactly like a Standby whose Active is healthy. +func TestPromote_DropsStandbyScopedSeries(t *testing.T) { + haRemoteLeaseAgeSeconds.Reset() + + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + _, _ = e.checkRemoteLeaseOnce(context.Background()) + require.Equal(t, 1, testutil.CollectAndCount(haRemoteLeaseAgeSeconds), + "precondition: the Standby was publishing a remote lease age") + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.True(t, promoted) + + assert.Equal(t, 0, testutil.CollectAndCount(haRemoteLeaseAgeSeconds), + "promotion must drop the remote lease age — there is no remote to age any more") + assert.Equal(t, 1, testutil.CollectAndCount(haLastPromotionTimestamp), + "and must start publishing when it promoted") +} + +func TestHAArmed_ZeroUntilTheActiveLeaseIsRead(t *testing.T) { + ctx := context.Background() + + // A standby whose remote reads always fail never arms. + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + assert.Equal(t, float64(0), testutil.ToFloat64(haArmed.WithLabelValues(string(ModeStandby))), + "construction must publish 0 so a never-arming standby is visible") + + _, _ = e.checkRemoteLeaseOnce(ctx) + assert.Equal(t, float64(0), testutil.ToFloat64(haArmed.WithLabelValues(string(ModeStandby))), + "a failed read must not arm the hub") + + // One successful read arms it. + remote := fakeClient(t) + lease := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now()) + require.NoError(t, remote.Create(ctx, lease)) + armed := NewClusterLeaderElector(fakeClient(t), remote, Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + _, err := armed.checkRemoteLeaseOnce(ctx) + require.NoError(t, err) + assert.Equal(t, float64(1), testutil.ToFloat64(haArmed.WithLabelValues(string(ModeStandby)))) +} + +func TestHARemoteLeaseReads_CountedByResult(t *testing.T) { + ctx := context.Background() + haRemoteLeaseReadsTotal.Reset() + + remote := fakeClient(t) + lease := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now()) + require.NoError(t, remote.Create(ctx, lease)) + ok := NewClusterLeaderElector(fakeClient(t), remote, Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + _, err := ok.checkRemoteLeaseOnce(ctx) + require.NoError(t, err) + + failing := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-c", Log: testLog(), + }) + _, _ = failing.checkRemoteLeaseOnce(ctx) + + assert.Equal(t, float64(1), testutil.ToFloat64(haRemoteLeaseReadsTotal.WithLabelValues(readResultOK))) + assert.Equal(t, float64(1), testutil.ToFloat64(haRemoteLeaseReadsTotal.WithLabelValues(readResultError))) +} + +// TestHARemoteLeaseAge_ClimbsWhileReadsFail is the property that makes this +// gauge a leading indicator rather than a lagging one. A failed read leaves the +// cached Lease in place by design, so the age must keep climbing against the +// clock — a gauge frozen at its last good value would report a healthy Active +// throughout the outage. +func TestHARemoteLeaseAge_ClimbsWhileReadsFail(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-90*time.Second)) + + _, _ = e.checkRemoteLeaseOnce(context.Background()) + + age := testutil.ToFloat64(haRemoteLeaseAgeSeconds.WithLabelValues(string(ModeStandby))) + assert.Greater(t, age, float64(85), "the age must reflect the retained stale lease, not the failed read") +} + +func TestRemoteLeaseAge(t *testing.T) { + now := time.Date(2026, 8, 10, 12, 0, 0, 0, time.UTC) + + _, ok := remoteLeaseAge(nil, now) + assert.False(t, ok, "no lease read yet is not an age of zero") + + noRenew := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", now) + noRenew.Spec.RenewTime = nil + _, ok = remoteLeaseAge(noRenew, now) + assert.False(t, ok, "a lease with no renewTime has no measurable age") + + past := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", now.Add(-30*time.Second)) + age, ok := remoteLeaseAge(past, now) + require.True(t, ok) + assert.Equal(t, 30*time.Second, age) + + // Clock skew: the Active's clock ahead of ours must clamp to zero rather + // than publish a negative age that would make an alert flap. + future := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", now.Add(10*time.Second)) + age, ok = remoteLeaseAge(future, now) + require.True(t, ok) + assert.Equal(t, time.Duration(0), age) +} + +// TestPromote_RecordsDurationTimestampAndSteps covers the metrics the promotion +// path exists to produce: when it happened, how long it took in total, and how +// long each bounded step took. +func TestPromote_RecordsDurationTimestampAndSteps(t *testing.T) { + haPromotionDurationSeconds.Reset() + haPromotionStepDurationSeconds.Reset() + haLastPromotionTimestamp.Reset() + + before := testutil.ToFloat64(haFailoverTotal) + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.True(t, promoted) + + assert.Equal(t, before+1, testutil.ToFloat64(haFailoverTotal)) + assert.Greater(t, testutil.ToFloat64(haLastPromotionTimestamp.WithLabelValues(string(ModeActive))), float64(0), + "a completed promotion must stamp when it happened") + + assert.Equal(t, uint64(1), + histogramSampleCount(t, haPromotionDurationSeconds, map[string]string{"outcome": outcomePromoted}), + "a successful promotion must be recorded as such, not as an abort") + assert.Equal(t, uint64(0), + histogramSampleCount(t, haPromotionDurationSeconds, map[string]string{"outcome": outcomeAborted})) + + // Every step of the sequence that ran must have timed itself. + for _, step := range []string{stepStopMirror, stepAcquireLease, stepPublishActive, stepKickReconcilers, stepEmitPromoted} { + assert.Equal(t, uint64(1), + histogramSampleCount(t, haPromotionStepDurationSeconds, map[string]string{"step": step}), + "step %s must record its own duration", step) + } +} + +// TestPromote_AbortIsLabelledAbortedAndCounted checks the other half of the +// outcome label. A guard refusal is not a failure and not a promotion; it has to +// be visible as its own thing, in both the duration histogram and the reason +// counter. +func TestPromote_AbortIsLabelledAbortedAndCounted(t *testing.T) { + haPromotionDurationSeconds.Reset() + haPromotionsAbortedTotal.Reset() + + // A standby whose Active is alive: the final-dial guard must refuse. + ctx := context.Background() + remote := fakeClient(t) + live := newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now()) + require.NoError(t, remote.Create(ctx, live)) + + e := NewClusterLeaderElector(fakeClient(t), remote, Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + promoted, err := e.promote(ctx) + require.NoError(t, err, "declining to promote is a correct outcome, not an error") + require.False(t, promoted) + + assert.Equal(t, uint64(1), + histogramSampleCount(t, haPromotionDurationSeconds, map[string]string{"outcome": outcomeAborted})) + assert.Equal(t, uint64(0), + histogramSampleCount(t, haPromotionDurationSeconds, map[string]string{"outcome": outcomePromoted})) + assert.Equal(t, float64(1), + testutil.ToFloat64(haPromotionsAbortedTotal.WithLabelValues(abortLeaseLive))) +} + +// TestPromote_ConcurrentRejectionIsNotTimed keeps the near-instant rejections +// out of the histogram. promote() returns in nanoseconds when the latch is +// already held, and a pile of those labelled "aborted" would drag the aborted +// quantiles to zero and hide the aborts that spent a whole grace period first. +func TestPromote_ConcurrentRejectionIsNotTimed(t *testing.T) { + haPromotionDurationSeconds.Reset() + haPromotionsAbortedTotal.Reset() + + e := standbyReadyToPromote(t) + e.promoting.Store(true) // pretend a concurrent tick holds the latch + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.False(t, promoted) + + assert.Equal(t, uint64(0), + histogramSampleCount(t, haPromotionDurationSeconds, map[string]string{"outcome": outcomeAborted}), + "a rejected concurrent attempt is not an attempt and must not be timed") + assert.Equal(t, float64(1), + testutil.ToFloat64(haPromotionsAbortedTotal.WithLabelValues(abortAlreadyPromoting)), + "it must still be counted, so the rejection is not invisible") +} + +func TestHALeaseRenewMetrics(t *testing.T) { + ctx := context.Background() + haLeaseLastRenewTime.Reset() + before := testutil.ToFloat64(haLeaseRenewErrorsTotal) + + e := NewClusterLeaderElector(fakeClient(t), nil, Options{ + Mode: ModeActive, Identity: "hub-a", Log: testLog(), + }) + require.NoError(t, e.renewOnce(ctx)) + assert.Greater(t, testutil.ToFloat64(haLeaseLastRenewTime.WithLabelValues(string(ModeActive))), float64(0), + "a successful renewal must stamp when it happened") + assert.Equal(t, before, testutil.ToFloat64(haLeaseRenewErrorsTotal)) + + failing := NewClusterLeaderElector(failingReadClient(t), nil, Options{ + Mode: ModeActive, Identity: "hub-a", Log: testLog(), + }) + require.Error(t, failing.renewOnce(ctx)) + assert.Equal(t, before+1, testutil.ToFloat64(haLeaseRenewErrorsTotal)) +} + +// TestHAPruneMetrics_CountResurrectionsAndStampTheRun covers the drift +// backstop's two signals. Prune re-enqueuing work is not reassurance: it means +// the informer path missed something, so zero is the healthy value and a steady +// non-zero rate is the actual finding. +func TestHAPruneMetrics_CountResurrectionsAndStampTheRun(t *testing.T) { + ctx := context.Background() + haPruneResurrectedTotal.Reset() + haPruneLastRunTimestamp.Reset() + + missing := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-missing"} + s := buildSyncer(t, newStubRemote()) + // Active has an object with no mirror on the Standby — the reverse diff. + s.remoteList = stubRemoteList([]syncKey{missing}, nil) + + s.pruneOnce(ctx) + + assert.Equal(t, float64(1), + testutil.ToFloat64(haPruneResurrectedTotal.WithLabelValues(testGVK.Kind)), + "an Active object with no mirror must be counted, not silently re-enqueued") + assert.Greater(t, testutil.ToFloat64(haPruneLastRunTimestamp.WithLabelValues(string(ModeStandby))), float64(0), + "a completed pass must stamp when it ran, so a stalled backstop is visible") +} + +// TestHAPruneLastRun_StampedEvenWhenAKindWasSkipped is the deliberate choice +// documented at the call site: a failed list skips that kind, but the pass did +// run, and the skip is already counted on ha_sync_errors_total. Not stamping +// would make a partially-degraded prune indistinguishable from one that never +// executed at all. +func TestHAPruneLastRun_StampedEvenWhenAKindWasSkipped(t *testing.T) { + haPruneLastRunTimestamp.Reset() + + s := buildSyncer(t, newStubRemote()) + s.remoteList = stubRemoteList(nil, fmt.Errorf("simulated transient list failure")) + + s.pruneOnce(context.Background()) + + assert.Greater(t, testutil.ToFloat64(haPruneLastRunTimestamp.WithLabelValues(string(ModeStandby))), float64(0)) +} + +// TestHASyncQueueDepth_RisesOnEnqueueAndFallsOnDrain is the distinction from +// ha_sync_lag_seconds: lag is only observed for items that completed, so a +// syncer falling behind reports healthy lag off the few that finish. Depth is +// what shows the backlog. +func TestHASyncQueueDepth_RisesOnEnqueueAndFallsOnDrain(t *testing.T) { + ctx := context.Background() + s := buildSyncer(t, newStubRemote()) + + s.enqueue(syncKey{GVK: testGVK, Namespace: "proj-a", Name: "one"}) + s.enqueue(syncKey{GVK: testGVK, Namespace: "proj-a", Name: "two"}) + assert.Equal(t, float64(2), testutil.ToFloat64(haSyncQueueDepth)) + + key, _ := s.queue.Get() + s.processOnce(ctx, key) + assert.Equal(t, float64(1), testutil.ToFloat64(haSyncQueueDepth), + "the gauge must fall as the backlog drains, not only ever rise") +} + +func contains(haystack []string, needle string) bool { + for _, s := range haystack { + if s == needle { + return true + } + } + return false +} diff --git a/pkg/ha/mirror.go b/pkg/ha/mirror.go new file mode 100644 index 000000000..3f74c7d4f --- /dev/null +++ b/pkg/ha/mirror.go @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// syncKey identifies one mirrored object by GVK and namespaced name. It is +// RemoteSyncer's workqueue item type (see remote_syncer.go); mirror.go only +// needs it to know what to read and where to write. +type syncKey struct { + GVK schema.GroupVersionKind + Namespace string + Name string +} + +// mirrorOp reports which write mirrorCreateOrUpdate actually performed (or +// "" if the conflict guard skipped it), so callers can label metrics without +// re-deriving it. +type mirrorOp string + +const ( + opCreate mirrorOp = "create" + opUpdate mirrorOp = "update" +) + +// mirrorCreateOrUpdate writes src onto the Standby via localClient: create if +// absent, update if present and syncer-owned. src is the informer's own +// cached object and is never mutated in place. +func mirrorCreateOrUpdate(ctx context.Context, localClient client.Client, key syncKey, res MirroredResource, src *unstructured.Unstructured) (mirrorOp, error) { + payload := src.DeepCopy() + payload.SetGroupVersionKind(key.GVK) + payload.SetResourceVersion("") + payload.SetUID("") + payload.SetManagedFields(nil) + payload.SetFinalizers(nil) + // While an Active-side object is Terminating (deletionTimestamp set, + // contents still being garbage-collected), the informer delivers it as + // an ordinary Update, not a Delete yet — copying that onto a Standby + // copy that isn't itself terminating fails immutable-field validation + // on a real API server (confirmed live: "field is immutable" on both + // deletionTimestamp and deletionGracePeriodSeconds). Its status is + // dropped from payload too, explicitly rather than relying on a real + // API server silently ignoring .status on the main resource endpoint: + // for at least one type (confirmed live: Namespace), a stray + // status.Phase="Terminating" is itself invalid once deletionTimestamp + // is empty. The Standby converges correctly once Active reports + // NotFound and mirrorDelete takes over; there's nothing useful to + // reflect about the in-between Terminating state. + if src.GetDeletionTimestamp() != nil { + delete(payload.Object, "status") + } + payload.SetDeletionTimestamp(nil) + payload.SetDeletionGracePeriodSeconds(nil) + if res.StripOwnerRefs { + payload.SetOwnerReferences(nil) + } + if res.Sanitize != nil { + res.Sanitize(payload) + } + + labels := payload.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[LabelSyncedFromActive] = LabelValueActive + payload.SetLabels(labels) + + annotations := payload.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[AnnotationSourceRV] = src.GetResourceVersion() + payload.SetAnnotations(annotations) + + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(key.GVK) + err := localClient.Get(ctx, types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, existing) + + var op mirrorOp + switch { + case apierrors.IsNotFound(err): + if err := localClient.Create(ctx, payload); err != nil { + return "", fmt.Errorf("creating mirror of %s %s/%s: %w", key.GVK.Kind, key.Namespace, key.Name, err) + } + op = opCreate + case err != nil: + return "", fmt.Errorf("reading existing target %s %s/%s: %w", key.GVK.Kind, key.Namespace, key.Name, err) + default: + if existing.GetLabels()[LabelSyncedFromActive] != LabelValueActive { + // Conflict guard: never overwrite an object the Standby didn't + // create itself. + return "", nil + } + if res.CreateOnly != nil && res.CreateOnly(src) { + // Seeded from the Active once, owned locally afterwards. The + // Standby's copy is deliberately allowed to diverge: for an + // SA-token shell it is the local token controller's write that + // makes the object useful, and re-applying the Active's version + // would undo it. AnnotationSourceRV therefore stays at whatever + // the source resourceVersion was at creation, which is harmless — + // nothing reads it back, it exists for operators debugging drift. + return "", nil + } + payload.SetResourceVersion(existing.GetResourceVersion()) + payload.SetUID(existing.GetUID()) + if err := localClient.Update(ctx, payload); err != nil { + return "", fmt.Errorf("updating mirror of %s %s/%s: %w", key.GVK.Kind, key.Namespace, key.Name, err) + } + op = opUpdate + } + + // Update() silently drops .status once a status subresource is + // registered — true for every entry in CRDMirrorSet. Mirror it + // explicitly, matching this repo's own UpdateStatus/CleanupUpdateStatus + // convention (util/reconciliation_utility.go, util/cleanup_utility.go). + // + // Skipped while src is itself Terminating: its status is about to become + // meaningless anyway, and for at least one type (confirmed live: + // Namespace) copying a Terminating status onto a payload whose + // deletionTimestamp was just stripped above fails a real API server + // validation rule — status.Phase can only be "Terminating" if + // deletionTimestamp is set. The Standby converges correctly once Active + // reports NotFound and mirrorDelete takes over. + if src.GetDeletionTimestamp() == nil { + if status, ok, _ := unstructured.NestedFieldNoCopy(src.Object, "status"); ok { + payload.Object["status"] = status + if err := localClient.Status().Update(ctx, payload); err != nil { + return op, fmt.Errorf("mirroring status of %s %s/%s: %w", key.GVK.Kind, key.Namespace, key.Name, err) + } + } + } + return op, nil +} + +// mirrorDelete removes the Standby's mirror of key, if the engine created it. +// Deleting an already-absent object is treated as success (idempotent — a +// concurrent prune pass may already have removed it). +func mirrorDelete(ctx context.Context, localClient client.Client, key syncKey) error { + existing := &unstructured.Unstructured{} + existing.SetGroupVersionKind(key.GVK) + err := localClient.Get(ctx, types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, existing) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("reading target %s %s/%s before delete: %w", key.GVK.Kind, key.Namespace, key.Name, err) + } + if existing.GetLabels()[LabelSyncedFromActive] != LabelValueActive { + return nil + } + if err := localClient.Delete(ctx, existing); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("deleting mirror of %s %s/%s: %w", key.GVK.Kind, key.Namespace, key.Name, err) + } + return nil +} diff --git a/pkg/ha/mirror_set.go b/pkg/ha/mirror_set.go new file mode 100644 index 000000000..164fba095 --- /dev/null +++ b/pkg/ha/mirror_set.go @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// Label and annotation keys the mirror engine stamps onto every object it +// writes to the Standby. LabelSyncedFromActive is the conflict guard: the +// engine only ever overwrites or deletes a Standby object that carries it, so +// it never touches anything the Standby's own reconcilers (or an operator) +// created directly — including a pre-existing Namespace such as kube-system +// or default, now that Namespace is an ordinary mirrored type (see +// CRDMirrorSet below) rather than a special-cased cold-start step. +const ( + LabelSyncedFromActive = "ha.kubeslice.io/synced-from" + LabelValueActive = "active" + AnnotationSourceRV = "ha.kubeslice.io/source-rv" +) + +// MirroredResource describes one GroupVersionKind RemoteSyncer mirrors from +// the Active hub to the Standby. +type MirroredResource struct { + GVK schema.GroupVersionKind + // StripOwnerRefs strips metadata.ownerReferences from the mirrored copy. + // Only VpnKeyRotation needs this: its ownerReference points at the + // Active-side SliceConfig's UID, which the Standby's mirrored SliceConfig + // does not share (a fresh UID is assigned on create) — left in place, the + // Standby's garbage collector would see a dangling reference and delete + // the object shortly after the mirror creates it. + StripOwnerRefs bool + // Skip, if set, excludes an object from mirroring based on its content. + Skip func(u *unstructured.Unstructured) bool + // CreateOnly, if set, reports objects the engine may create on the Standby + // but must never overwrite once they exist there — the Standby's copy is + // seeded from the Active and then owned locally. See the Secret row in + // CredentialMirrorSet for the case this exists for. + CreateOnly func(u *unstructured.Unstructured) bool + // Sanitize, if set, strips fields from the payload after the engine's own + // normalisation and before the write. It receives the payload copy, never + // the informer's cached object. + Sanitize func(payload *unstructured.Unstructured) + // RequireMirroredNamespace restricts mirroring to objects whose namespace + // the syncer itself mirrors — i.e. the namespace is present in the remote + // cache's label-scoped Namespace view (see namespaceMirrorSelector). Set + // on every credential row: core types exist in every namespace, and no + // name-based rule can draw this boundary safely — under the Helm chart's + // real-world --project-namespace-prefix ("kubeslice-"), the controller's + // own kubeslice-controller namespace matches the project-namespace naming + // pattern, and a prefix test would have mirrored its webhook TLS key, + // image-pull credentials, and Helm release Secrets onto the Standby + // (found live against a Helm-installed Active hub). The label boundary is + // the one ReconcileProjectNamespace actually maintains. + RequireMirroredNamespace bool +} + +const ( + groupController = "controller.kubeslice.io" + groupWorker = "worker.kubeslice.io" + groupRBAC = "rbac.authorization.k8s.io" +) + +func gvk(group, kind string) schema.GroupVersionKind { + return schema.GroupVersionKind{Group: group, Version: "v1alpha1", Kind: kind} +} + +// CRDMirrorSet is the set of hub-side resources mirrored Active -> Standby. +// +// This intentionally does not match issue #295's own CRD table, which names +// Slice/SliceGateway/ServiceExport — none of those types exist in this repo. +// They are worker-cluster data-plane CRDs (group networking.kubeslice.io) +// owned by the separate worker-operator repo, irrelevant to hub-to-hub +// mirroring. Verified against apis/controller/v1alpha1 and apis/worker/v1alpha1. +var CRDMirrorSet = []MirroredResource{ + {GVK: schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}}, + {GVK: gvk(groupController, "Project")}, + {GVK: gvk(groupController, "Cluster")}, + {GVK: gvk(groupController, "SliceConfig")}, + {GVK: gvk(groupController, "ServiceExportConfig")}, + {GVK: gvk(groupController, "SliceQoSConfig")}, + {GVK: gvk(groupController, "VpnKeyRotation"), StripOwnerRefs: true}, + {GVK: gvk(groupWorker, "WorkerSliceConfig")}, + {GVK: gvk(groupWorker, "WorkerSliceGateway")}, + {GVK: gvk(groupWorker, "WorkerServiceImport")}, +} + +// isServiceAccountTokenSecret reports whether u is a +// kubernetes.io/service-account-token Secret. +func isServiceAccountTokenSecret(u *unstructured.Unstructured) bool { + secretType, _, _ := unstructured.NestedString(u.Object, "type") + return secretType == string(corev1.SecretTypeServiceAccountToken) +} + +// sanitizeSecret reduces a service-account-token Secret to its shell before +// the mirror writes it: everything that identifies which ServiceAccount the +// token is for, and nothing that is (or names) the token itself. Other Secret +// types pass through untouched. +// +// Two things have to go, for unrelated reasons: +// +// - .data, because an SA token is signed by the issuing cluster's own +// service-account key and is therefore cryptographically invalid on the +// Standby. Mirroring it would ship a credential that silently fails to +// authenticate — worse than none, since its presence masks the absence of +// a real one. The Standby's own token controller populates the empty +// shell with a locally valid token, which is the entire point of carrying +// the shell across. +// - the kubernetes.io/service-account.uid annotation, which the token +// controller adds when it populates the Active's copy and then validates +// against the local ServiceAccount's UID, deleting the Secret on +// mismatch. A mirrored ServiceAccount is created fresh on the Standby and +// never shares the Active's UID (see mirrorCreateOrUpdate, which clears +// it), so a copied annotation never matches: the Standby's token +// controller would delete the Secret, prune's reverse diff would restore +// it, and the two would chase each other indefinitely. +// +// The kubernetes.io/service-account.name annotation is deliberately kept — it +// is how the token controller knows which account to mint for, and it is the +// only link between the shell and the mirrored ServiceAccount. +func sanitizeSecret(payload *unstructured.Unstructured) { + if !isServiceAccountTokenSecret(payload) { + return + } + delete(payload.Object, "data") + annotations := payload.GetAnnotations() + if annotations == nil { + return + } + delete(annotations, corev1.ServiceAccountUIDKey) + payload.SetAnnotations(annotations) +} + +// FullMirrorSet is everything a production Standby mirrors: CRDMirrorSet +// plus CredentialMirrorSet. Returned as a fresh slice so callers cannot +// mutate the package-level tables through it. +func FullMirrorSet() []MirroredResource { + full := make([]MirroredResource, 0, len(CRDMirrorSet)+len(CredentialMirrorSet)) + full = append(full, CRDMirrorSet...) + return append(full, CredentialMirrorSet...) +} + +// CredentialMirrorSet is the set of credential resources mirrored Active -> +// Standby so a promoted Standby can serve its worker clusters without manual +// re-provisioning: worker-identity RBAC (Role/RoleBinding/ServiceAccount, the +// only RBAC kinds access_control_service.go ever creates — no ClusterRole or +// ClusterRoleBinding, despite ADR #293 Decision 6's broader wording) and Secrets: +// the gateway certificates the ovpn job generates, and the shells of the +// worker service-account token Secrets. +// +// Every row sets RequireMirroredNamespace — see that field's doc comment for +// why the boundary is the mirrored-namespace set and not a name pattern — +// and StripOwnerRefs: ownerReferences resolve by UID, which never survives +// the cross-cluster copy, and unlike the CRD set (audited — only +// VpnKeyRotation ever gets a reference, from this repo's own code) +// credential objects are also written by actors outside this repo (the +// token controller, the cert-generator job), so no such audit can hold here. +var CredentialMirrorSet = []MirroredResource{ + // Secrets cover two unrelated cases. Gateway certificates and the like are + // ordinary mirrored objects. Service-account-token Secrets are carried as + // empty shells (Sanitize) that the Standby's own token controller fills + // in, which is what gives a Standby a worker credential valid on itself + // before any failover has happened — see ADR #293 Decision 6. Those shells + // are CreateOnly because the engine's update path + // is an unconditional full write: a payload with no .data would clear the + // locally minted token on every informer resync, and the token controller + // would mint a fresh one, invalidating whatever copy a worker had already + // been given. Seed it once and leave it alone. + { + GVK: schema.GroupVersionKind{Version: "v1", Kind: "Secret"}, + StripOwnerRefs: true, + CreateOnly: isServiceAccountTokenSecret, + Sanitize: sanitizeSecret, + RequireMirroredNamespace: true, + }, + { + GVK: schema.GroupVersionKind{Version: "v1", Kind: "ServiceAccount"}, + StripOwnerRefs: true, + RequireMirroredNamespace: true, + }, + { + GVK: schema.GroupVersionKind{Group: groupRBAC, Version: "v1", Kind: "Role"}, + StripOwnerRefs: true, + RequireMirroredNamespace: true, + }, + { + GVK: schema.GroupVersionKind{Group: groupRBAC, Version: "v1", Kind: "RoleBinding"}, + StripOwnerRefs: true, + RequireMirroredNamespace: true, + }, +} diff --git a/pkg/ha/mirror_test.go b/pkg/ha/mirror_test.go new file mode 100644 index 000000000..e69708898 --- /dev/null +++ b/pkg/ha/mirror_test.go @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var testGVK = schema.GroupVersionKind{Group: groupController, Version: "v1alpha1", Kind: "SliceConfig"} + +func newTestUnstructured(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(gvk) + u.SetNamespace(namespace) + u.SetName(name) + return u +} + +// mirrorFakeClient builds a fake client that emulates the status-subresource +// split real clusters apply to every CRDMirrorSet entry: Update() must not +// alter .status, only Status().Update() may. +func mirrorFakeClient(t *testing.T, objs ...client.Object) client.Client { + t.Helper() + statusGVKObj := &unstructured.Unstructured{} + statusGVKObj.SetGroupVersionKind(testGVK) + return fake.NewClientBuilder(). + WithScheme(testScheme(t)). + WithStatusSubresource(statusGVKObj). + WithObjects(objs...). + Build() +} + +func getUnstructured(t *testing.T, c client.Client, key syncKey) *unstructured.Unstructured { + t.Helper() + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(key.GVK) + require.NoError(t, c.Get(context.Background(), types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, got)) + return got +} + +func TestMirrorCreateOrUpdate_CreatesWithLabelAndAnnotation(t *testing.T) { + ctx := context.Background() + c := mirrorFakeClient(t) + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + src.SetResourceVersion("999") + + op, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err) + assert.Equal(t, opCreate, op) + + got := getUnstructured(t, c, key) + assert.Equal(t, LabelValueActive, got.GetLabels()[LabelSyncedFromActive]) + assert.Equal(t, "999", got.GetAnnotations()[AnnotationSourceRV]) +} + +func TestMirrorCreateOrUpdate_UpdatesExistingSyncedObject(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + existing := newTestUnstructured(testGVK, key.Namespace, key.Name) + existing.SetLabels(map[string]string{LabelSyncedFromActive: LabelValueActive}) + existing.SetResourceVersion("1") + existing.SetUID("standby-uid-1") + c := mirrorFakeClient(t, existing) + + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, "updated-value", "spec", "field")) + src.SetResourceVersion("active-rv-2") + + op, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err) + assert.Equal(t, opUpdate, op) + + got := getUnstructured(t, c, key) + val, _, _ := unstructured.NestedString(got.Object, "spec", "field") + assert.Equal(t, "updated-value", val) + assert.Equal(t, "active-rv-2", got.GetAnnotations()[AnnotationSourceRV], + "source-rv annotation should reflect the Active object's resourceVersion") +} + +func TestMirrorCreateOrUpdate_ConflictGuardSkipsUnlabeledExisting(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + existing := newTestUnstructured(testGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(existing.Object, "hand-applied", "spec", "field")) + c := mirrorFakeClient(t, existing) + + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, "from-active", "spec", "field")) + + op, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err) + assert.Equal(t, mirrorOp(""), op, "conflict guard should report no-op") + + got := getUnstructured(t, c, key) + val, _, _ := unstructured.NestedString(got.Object, "spec", "field") + assert.Equal(t, "hand-applied", val, "an unlabeled existing object must never be overwritten") +} + +func TestMirrorCreateOrUpdate_StripOwnerRefsOnlyWhenConfigured(t *testing.T) { + ctx := context.Background() + ownerRef := metav1.OwnerReference{APIVersion: "v1alpha1", Kind: "SliceConfig", Name: "owner", UID: "owner-uid"} + + stripKey := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "vpn-1"} + src := newTestUnstructured(testGVK, stripKey.Namespace, stripKey.Name) + src.SetOwnerReferences([]metav1.OwnerReference{ownerRef}) + c := mirrorFakeClient(t) + + _, err := mirrorCreateOrUpdate(ctx, c, stripKey, MirroredResource{GVK: testGVK, StripOwnerRefs: true}, src) + require.NoError(t, err) + got := getUnstructured(t, c, stripKey) + assert.Empty(t, got.GetOwnerReferences(), "StripOwnerRefs=true must strip ownerReferences") + + keepKey := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "vpn-2"} + src2 := newTestUnstructured(testGVK, keepKey.Namespace, keepKey.Name) + src2.SetOwnerReferences([]metav1.OwnerReference{ownerRef}) + + _, err = mirrorCreateOrUpdate(ctx, c, keepKey, MirroredResource{GVK: testGVK, StripOwnerRefs: false}, src2) + require.NoError(t, err) + got2 := getUnstructured(t, c, keepKey) + assert.Len(t, got2.GetOwnerReferences(), 1, "StripOwnerRefs=false must leave ownerReferences untouched") +} + +func TestMirrorCreateOrUpdate_StripsDeletionTimestampFromTerminatingSource(t *testing.T) { + ctx := context.Background() + gracePeriod := int64(0) + terminating := func(u *unstructured.Unstructured) { + now := metav1.Now() + u.SetDeletionTimestamp(&now) + u.SetDeletionGracePeriodSeconds(&gracePeriod) + } + + t.Run("update path", func(t *testing.T) { + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + existing := newTestUnstructured(testGVK, key.Namespace, key.Name) + existing.SetLabels(map[string]string{LabelSyncedFromActive: LabelValueActive}) + c := mirrorFakeClient(t, existing) + + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + terminating(src) + + op, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err, "an Active-side object mid-Terminating must not fail mirroring") + assert.Equal(t, opUpdate, op) + + got := getUnstructured(t, c, key) + assert.Nil(t, got.GetDeletionTimestamp(), "the Standby mirror must never carry a deletionTimestamp copied from a Terminating Active-side object") + assert.Nil(t, got.GetDeletionGracePeriodSeconds()) + }) + + t.Run("create path", func(t *testing.T) { + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-2"} + c := mirrorFakeClient(t) + + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + terminating(src) + + op, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err, "creating a mirror from an already-Terminating source must not fail") + assert.Equal(t, opCreate, op) + + got := getUnstructured(t, c, key) + assert.Nil(t, got.GetDeletionTimestamp()) + }) +} + +func TestMirrorCreateOrUpdate_MirrorsStatusExplicitly(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + require.NoError(t, unstructured.SetNestedField(src.Object, "Ready", "status", "phase")) + c := mirrorFakeClient(t) + + _, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err) + + got := getUnstructured(t, c, key) + phase, ok, _ := unstructured.NestedString(got.Object, "status", "phase") + require.True(t, ok, "status.phase must be present after mirroring — a plain Update() alone would have dropped it") + assert.Equal(t, "Ready", phase) +} + +func TestMirrorCreateOrUpdate_SkipsStatusMirrorWhenSourceIsTerminating(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + c := mirrorFakeClient(t) + + src := newTestUnstructured(testGVK, key.Namespace, key.Name) + now := metav1.Now() + src.SetDeletionTimestamp(&now) + require.NoError(t, unstructured.SetNestedField(src.Object, "Terminating", "status", "phase")) + + op, err := mirrorCreateOrUpdate(ctx, c, key, MirroredResource{GVK: testGVK}, src) + require.NoError(t, err, "mirroring a Terminating source must not fail trying to write an invalid status combination") + assert.Equal(t, opCreate, op) + + got := getUnstructured(t, c, key) + _, ok, _ := unstructured.NestedString(got.Object, "status", "phase") + assert.False(t, ok, "status must not be mirrored while the source is Terminating — see mirror.go for the real API-server validation rule this avoids") +} + +func TestMirrorDelete_IdempotentOnNotFound(t *testing.T) { + c := mirrorFakeClient(t) + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "missing"} + assert.NoError(t, mirrorDelete(context.Background(), c, key)) +} + +func TestMirrorDelete_DeletesSyncedObject(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + existing := newTestUnstructured(testGVK, key.Namespace, key.Name) + existing.SetLabels(map[string]string{LabelSyncedFromActive: LabelValueActive}) + c := mirrorFakeClient(t, existing) + + require.NoError(t, mirrorDelete(ctx, c, key)) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(key.GVK) + err := c.Get(ctx, types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, got) + assert.True(t, apierrors.IsNotFound(err), "object should be gone after delete") +} + +func TestMirrorDelete_ConflictGuardSkipsUnlabeledExisting(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + existing := newTestUnstructured(testGVK, key.Namespace, key.Name) + c := mirrorFakeClient(t, existing) + + require.NoError(t, mirrorDelete(ctx, c, key)) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(key.GVK) + err := c.Get(ctx, types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, got) + assert.NoError(t, err, "an unlabeled existing object must never be deleted") +} diff --git a/pkg/ha/mode.go b/pkg/ha/mode.go new file mode 100644 index 000000000..bf7e28297 --- /dev/null +++ b/pkg/ha/mode.go @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package ha implements cross-cluster (Active/Standby) high availability for the +// kubeslice-controller. It coordinates leadership between two separate hub +// clusters so that only the Active hub writes to worker clusters. This is +// distinct from controller-runtime's in-cluster --leader-elect, which only +// coordinates multiple pods sharing a single API server. See ADR #293. +package ha + +import ( + "fmt" + "strings" +) + +// HAMode identifies the high-availability role this controller instance plays. +type HAMode string + +const ( + // ModeActive means this controller holds (or competes for) leadership by + // renewing a Lease on its own cluster; while it is the leader its reconcilers + // write to worker clusters. + ModeActive HAMode = "active" + // ModeStandby means this controller mirrors the Active hub and never writes. + // In issue #294 a Standby watches the Active's Lease but does not promote; + // promotion is issue #297. + ModeStandby HAMode = "standby" + // ModeStandalone is the default single-hub behaviour: always the leader, no + // Lease and no remote watching — identical to the controller before HA. + ModeStandalone HAMode = "standalone" +) + +// ParseHAModeStrict converts a flag/env string into an HAMode, rejecting a +// non-empty value that names no known mode. Empty still means ModeStandalone, +// which is what every deployment that passes no --ha-mode at all gets. +// +// Coercing a typo to standalone looks like failing safe and is the opposite. +// Standalone is unconditionally the leader, so a hub started with +// --ha-mode=stanby does not become an inert Standby: it becomes a second +// unfenced writer alongside the real Active, reconciling the same worker +// clusters. That is the dual-writer state the whole design exists to prevent, +// reached silently, from one transposed letter in a chart value. A hub that +// refuses to start is recoverable in a way that one is not. +func ParseHAModeStrict(s string) (HAMode, error) { + trimmed := strings.TrimSpace(s) + if trimmed == "" { + return ModeStandalone, nil + } + mode := HAMode(strings.ToLower(trimmed)) + if !mode.IsValid() { + return ModeStandalone, fmt.Errorf( + "unknown --ha-mode %q: expected %q, %q or %q", s, ModeActive, ModeStandby, ModeStandalone) + } + return mode, nil +} + +// ParseHAMode converts a flag/env string into an HAMode, mapping empty or +// unrecognised values to ModeStandalone. Callers acting on operator input +// should prefer ParseHAModeStrict, which reports the unrecognised value instead +// of guessing; see its comment for why guessing is unsafe here. +func ParseHAMode(s string) HAMode { + switch HAMode(strings.ToLower(strings.TrimSpace(s))) { + case ModeActive: + return ModeActive + case ModeStandby: + return ModeStandby + default: + return ModeStandalone + } +} + +// IsValid reports whether m is one of the known modes. +func (m HAMode) IsValid() bool { + switch m { + case ModeActive, ModeStandby, ModeStandalone: + return true + default: + return false + } +} diff --git a/pkg/ha/mode_test.go b/pkg/ha/mode_test.go new file mode 100644 index 000000000..3a240da2c --- /dev/null +++ b/pkg/ha/mode_test.go @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import "testing" + +func TestParseHAMode(t *testing.T) { + cases := map[string]HAMode{ + "active": ModeActive, + "ACTIVE": ModeActive, + " standby ": ModeStandby, + "standalone": ModeStandalone, + "": ModeStandalone, + "garbage": ModeStandalone, + } + for in, want := range cases { + if got := ParseHAMode(in); got != want { + t.Errorf("ParseHAMode(%q) = %q, want %q", in, got, want) + } + } +} + +func TestHAModeIsValid(t *testing.T) { + for _, m := range []HAMode{ModeActive, ModeStandby, ModeStandalone} { + if !m.IsValid() { + t.Errorf("%q should be valid", m) + } + } + if HAMode("nope").IsValid() { + t.Error("unknown mode should be invalid") + } +} + +// TestParseHAModeStrict_RejectsTypos covers the case that makes lenient parsing +// dangerous: standalone is unconditionally the leader, so a hub whose --ha-mode +// was mistyped does not fail closed into an inert Standby, it fails OPEN into a +// second unfenced writer against the same worker clusters as the real Active. +func TestParseHAModeStrict_RejectsTypos(t *testing.T) { + for _, bad := range []string{"stanby", "activ", "primary", "true", "STANDBYY"} { + mode, err := ParseHAModeStrict(bad) + if err == nil { + t.Errorf("ParseHAModeStrict(%q) must reject an unknown mode, got %q", bad, mode) + } + } +} + +// TestParseHAModeStrict_AcceptsKnownModesAndEmpty pins the other half: every +// deployment that passes no --ha-mode at all must keep getting standalone, and +// the documented spellings must survive surrounding whitespace and case. +func TestParseHAModeStrict_AcceptsKnownModesAndEmpty(t *testing.T) { + for in, want := range map[string]HAMode{ + "": ModeStandalone, + " ": ModeStandalone, + "standalone": ModeStandalone, + "active": ModeActive, + "standby": ModeStandby, + " Standby ": ModeStandby, + "ACTIVE": ModeActive, + } { + got, err := ParseHAModeStrict(in) + if err != nil { + t.Errorf("ParseHAModeStrict(%q) returned an error: %v", in, err) + continue + } + if got != want { + t.Errorf("ParseHAModeStrict(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/pkg/ha/promotion.go b/pkg/ha/promotion.go new file mode 100644 index 000000000..8b15045c1 --- /dev/null +++ b/pkg/ha/promotion.go @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "time" + + coordinationv1 "k8s.io/api/coordination/v1" +) + +// PromotionHooks are the effects promotion has outside the elector itself. +// They are injected rather than imported so the elector stays independent of +// the mirror, the publisher and the controller-runtime manager — and so the +// whole sequence is testable without any of them. +// +// Every hook is optional; a nil hook is skipped. That is what lets a Standby +// run the sequence in a unit test, and it is why the elector needs no knowledge +// of what a RemoteSyncer or a reconciler even is. +type PromotionHooks struct { + // StopMirror cancels the RemoteSyncer and must not return until it has + // fully stopped writing, or until ctx expires — whichever comes first. + // Returning early, before the mirror has confirmed it stopped, is what step + // 3 of promote exists to prevent; returning an error on expiry is correct + // and aborts the promotion. Bounded by promotionGracePeriod. + StopMirror func(ctx context.Context) error + + // PublishActiveController writes status.activeController on this hub's own + // Cluster CRs, so workers can discover the new Active. Bounded by + // promotionGracePeriod; on expiry promotion proceeds anyway. + PublishActiveController func(ctx context.Context) error + + // KickReconcilers re-enqueues every object of every reconciled type. The + // write fence drops rather than requeues while a hub is Standby, so without + // this a promoted hub reconciles nothing that already existed until the + // informer resync period (10h by default). + KickReconcilers func(ctx context.Context) error + + // EmitPromotedEvent records the promotion as a Kubernetes Event against the + // newly-acquired Lease. + EmitPromotedEvent func(ctx context.Context, lease *coordinationv1.Lease) error +} + +// promote runs the full Standby -> Active sequence. It reports whether +// leadership was actually taken; a guard refusal returns (false, nil), because +// declining to promote is a correct outcome and not an error. +// +// The order below is deliberate, and two steps of it are load-bearing enough to +// state up front — the sequence sketched in issue #297 has them in an order that +// opens the write fence while the mirror may still be writing: +// +// 0. take the promotion latch — the write fence is held SHUT from here +// 1. (caller) staleness verdict +// 2. guards: self-health, then the final bounded dial +// 3. STOP the mirror, and WAIT for it to confirm it stopped +// 4. (caller) stop watching the Active's Lease +// 5. acquire the Lease on THIS hub's own cluster +// 6. mode = Active +// 7. publish status.activeController [budget: promotionGracePeriod] +// 8. release the latch and take leadership — THE WRITE FENCE OPENS HERE +// 9. start renewing our own Lease +// 10. re-enqueue every object of every reconciled type +// 11. emit PromotedToActive and count the failover +func (e *ClusterLeaderElector) promote(ctx context.Context) (bool, error) { + // Step 0. One-way and once-only, in two parts. + // + // The latch stops two attempts interleaving. But it is released on success, + // so it cannot by itself stop a second attempt on an already-promoted hub — + // and that is not merely redundant, it is unsafe: promotion has by then + // started a renewal loop that owns lastRenew and is actively writing the + // Lease, so a re-run both races that goroutine and fights it for the same + // object. (-race caught exactly this.) A hub that is already Active is + // already in the state promotion produces, so report success and do nothing. + if e.Mode() == ModeActive { + e.log.Debugw("already active; nothing to promote") + return true, nil + } + if !e.promoting.CompareAndSwap(false, true) { + e.abortPromotion(ctx, abortAlreadyPromoting) + e.log.Warnw("promotion already in progress; ignoring concurrent attempt") + return false, nil + } + // Re-check under the latch: two callers could both have passed the mode + // check above before either took it. + if e.Mode() == ModeActive { + e.promoting.Store(false) + return true, nil + } + // Until step 8 succeeds, every exit path returns the hub to a fenced, + // still-armed Standby that is free to try again next tick — with one + // deliberate exception, called out here because it is otherwise invisible. + // + // Step 3 stops the state mirror, and nothing in this package can start it + // again: main.go owns the syncer's lifecycle and hands promotion only a + // one-way StopMirror hook. So an abort *after* step 3 leaves a Standby that + // no longer mirrors. Every later attempt still works — StopMirror is + // idempotent and returns immediately once the syncer has exited, so a hub + // whose local API server was briefly unwritable promotes on a subsequent + // tick. The bad case is an abort after step 3 followed by the Active + // recovering: the guards then correctly refuse to promote, and this hub + // stays a Standby whose mirror is dead, drifting further from the Active + // until the process restarts. + // + // That is narrow — it needs the local Lease write to fail in the window + // between the guards passing and the Active coming back — but it is silent, + // so it is logged at error level rather than left to be discovered from a + // stale mirror much later. Restarting the mirror in place would mean giving + // promotion a two-way handle on a component it deliberately does not own. + // + // Timed from here rather than from the top of the function, so that the two + // early returns above stay out of the histogram. Neither is an attempt: an + // already-Active hub and a hub whose latch is held by a concurrent tick both + // return in nanoseconds, and a pile of near-zero observations labelled + // "aborted" would drag the aborted quantiles toward zero and hide the abort + // that actually matters — the one that spent a whole grace period first. + start := time.Now() + promoted := false + mirrorStopped := false + defer func() { + outcome := outcomeAborted + if promoted { + outcome = outcomePromoted + } + haPromotionDurationSeconds.WithLabelValues(outcome).Observe(time.Since(start).Seconds()) + if promoted { + return + } + e.promoting.Store(false) + if mirrorStopped { + e.log.Errorw("promotion aborted after the state mirror was stopped; this hub is a " + + "standby that is no longer mirroring the active hub, and will not resume until it " + + "promotes or the process restarts") + } + }() + + // A precondition rather than a guard, stated because it is otherwise + // implicit: without a client to the Active there is no way to have observed + // it alive, so there is nothing that could justify concluding it is gone. + // WatchRemoteLease refuses to start without one and the arming rule cannot + // be satisfied without one either, so this is unreachable from the loop — + // but promote is a method, and a future caller (a forced-promotion path, for + // instance) would otherwise crash inside the final dial rather than be told + // no. + if e.remoteClient == nil { + e.abortPromotion(ctx, abortNoRemoteClient) + return false, fmt.Errorf("cannot promote without a client to the active hub") + } + + e.log.Infow("promotion sequence starting", "identity", e.identity, "lease", e.leaseName) + + // Step 2. + if !e.guardsAllowPromotion(ctx) { + return false, nil + } + + // Step 3. This must complete before the write fence opens, and it is a real + // dual-writer bug rather than a stylistic preference. The most common + // trigger for a promotion is the Active's controller pod dying while its API + // server stays perfectly healthy — in which case the mirror's informers are + // still live and still mirroring at this exact moment. Every mirrored object + // carries the syncer's own label, and the mirror's conflict guard only skips + // objects WITHOUT it, so the mirror is entitled to overwrite precisely the + // objects a promoted hub's reconcilers are about to write. Worse, prune's + // reverse diff re-enqueues Active-side objects missing locally, so anything + // the new Active legitimately deletes gets resurrected within one sync + // interval. Opening the fence first means promoting into a hub that is + // fighting itself. + // + // The wait is bounded, and that bound matters more than it looks. Waiting + // indefinitely is the tempting choice, because proceeding without a stopped + // mirror is exactly the dual-writer state above. But the watch loop calls + // promote synchronously, so an unbounded wait on a mirror that never exits + // blocks the loop: no further polls, no further staleness evaluation, no + // failover ever, and nothing logged after the sequence started. Choosing + // "never promote into a dual writer" that way silently buys "never promote + // at all", which is strictly worse. Bounded, expiry aborts the attempt + // loudly and the next tick retries — so a mirror that is merely slow costs + // one tick, and a mirror that is genuinely stuck is visible instead of mute. + if e.hooks.StopMirror != nil { + stopCtx, cancel := context.WithTimeout(ctx, e.promotionGracePeriod) + // Set before the call, not after: cancellation has already been + // delivered by the time this returns, so a timeout leaves the mirror + // stopping regardless of the error. + mirrorStopped = true + stepStart := time.Now() + err := e.hooks.StopMirror(stopCtx) + cancel() + observeStep(stepStopMirror, stepStart) + if err != nil { + // Abort rather than continue. A mirror that has not confirmed it + // stopped is the dual-writer state this step exists to prevent, so + // the safe move is to stay a fenced Standby and retry next tick. + e.abortPromotion(ctx, abortMirrorNotStopped) + return false, fmt.Errorf("stopping the state mirror before promotion: %w", err) + } + e.log.Infow("state mirror stopped and confirmed exited") + } + + // Step 5. Note where the freshness judgement is NOT: acquireOrRenewLease + // takes the Lease unconditionally, which stays correct here because this + // Lease lives on the promoting hub's own cluster, where last-writer-wins is + // the right rule. Judging whether the *other* hub is still alive is the + // remote read's job, and it already happened in step 2. + acquireStart := time.Now() + lease, err := acquireOrRenewLease(ctx, e.localClient, e.leaseName, e.leaseNS, e.identity, e.leaseDuration) + observeStep(stepAcquireLease, acquireStart) + if err != nil { + e.abortPromotion(ctx, abortLeaseAcquireFailed) + return false, fmt.Errorf("acquiring the lease on this hub: %w", err) + } + e.lastRenew = time.Now() + haLeaseLastRenewTime.WithLabelValues(string(ModeActive)).Set(float64(e.lastRenew.Unix())) + e.log.Infow("acquired lease on this hub", "lease", e.leaseName, "namespace", e.leaseNS) + + // Step 6. + e.setMode(ModeActive) + + // Step 7. A budget, not a precondition: a hub that cannot publish is still a + // better Active than no Active at all, and the publisher's own loop keeps + // retrying afterwards. Failing here would strand the cluster with no writer. + if e.hooks.PublishActiveController != nil { + pubCtx, cancel := context.WithTimeout(ctx, e.promotionGracePeriod) + stepStart := time.Now() + err := e.hooks.PublishActiveController(pubCtx) + cancel() + observeStep(stepPublishActive, stepStart) + if err != nil { + haActivePublishErrorsTotal.Inc() + e.log.Errorw("could not publish activeController within the promotion grace period; "+ + "continuing anyway, the publisher loop will retry", + "error", err, "gracePeriod", e.promotionGracePeriod) + } else { + e.log.Infow("published activeController for the new Active") + } + } + + // Step 8. Steps 0 and 8 bracket everything above, so the fence stayed shut + // for the entire sequence — which is what gives "step 7 completes before the + // reconcilers are live" real teeth without inventing any external status + // surface. + e.promoting.Store(false) + e.setLeader(true) + promoted = true + + // Step 9. StartLeaseRenewal blocks until ctx is done, so it owns a goroutine. + // It is safe to start only now: it checks the mode, which step 6 has set. + go func() { + if err := e.StartLeaseRenewal(ctx); err != nil { + e.log.Errorw("lease renewal loop exited after promotion", "error", err) + } + }() + + // Step 10. Not optional. The fence drops reconcile requests rather than + // requeuing them, so flipping it causes no reconcile at all: everything the + // Standby ignored is gone, not parked, and nothing fires again until an + // object changes or the informer resyncs. Mirrored objects carry no + // finalizers until a reconciler re-adds them, so until this runs, deleting a + // SliceConfig on the promoted hub skips deboarding entirely. + // + // Ordered after step 8 on purpose: a kick delivered while the fence was + // still shut would be dropped without requeue, which is the exact failure it + // exists to fix. + // + // Bounded like the steps before it, for a reason specific to how the kick + // will be implemented: it pushes one event per object into a channel per + // reconciled type, and those channels are only drained once the manager is + // running. main.go starts this watch loop before mgr.Start, so a kick that + // arrives in that window has nothing reading the other end. Unbounded, it + // would block here forever on an already-promoted hub — leadership taken, + // fence open, and the promotion never finishing or reporting itself. + if e.hooks.KickReconcilers != nil { + kickCtx, cancel := context.WithTimeout(ctx, e.promotionGracePeriod) + stepStart := time.Now() + err := e.hooks.KickReconcilers(kickCtx) + cancel() + observeStep(stepKickReconcilers, stepStart) + if err != nil { + e.log.Errorw("could not re-enqueue objects after promotion; pre-existing state may "+ + "stay unreconciled until the informer resync period", "error", err) + } else { + e.log.Infow("re-enqueued all reconciled types after promotion") + } + } + + // Step 11. + haFailoverTotal.Inc() + haLastPromotionTimestamp.WithLabelValues(string(ModeActive)).Set(float64(time.Now().Unix())) + // Drop the Standby-scoped series. This hub no longer watches a remote hub, so + // leaving them behind would publish a frozen remote-lease age for a remote it + // has stopped reading -- indistinguishable, to an age-based alert, from a + // Standby whose Active is perfectly healthy. + haRemoteLeaseAgeSeconds.DeleteLabelValues(string(ModeStandby)) + // Detection is measured to `start`, not to now: it is the time from the + // Active's last observed proof of life to the moment this hub committed to + // taking over, and everything after `start` is the sequence's own cost, which + // ha_promotion_duration_seconds already reports. Keeping them disjoint is + // what lets the two be added into a total no-writer window. + if age, ok := remoteLeaseAge(e.lastSeenLease, start); ok { + haFailoverDetectionSeconds.Observe(age.Seconds()) + } + if e.hooks.EmitPromotedEvent != nil { + eventCtx, cancel := context.WithTimeout(ctx, e.promotionGracePeriod) + stepStart := time.Now() + err := e.hooks.EmitPromotedEvent(eventCtx, lease) + cancel() + observeStep(stepEmitPromoted, stepStart) + if err != nil { + e.log.Errorw("could not emit PromotedToActive event", "error", err) + } + } + e.log.Infow("PROMOTED to active", "identity", e.identity, "lease", e.leaseName) + return true, nil +} diff --git a/pkg/ha/promotion_event.go b/pkg/ha/promotion_event.go new file mode 100644 index 000000000..e8542da75 --- /dev/null +++ b/pkg/ha/promotion_event.go @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + coordinationv1 "k8s.io/api/coordination/v1" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" +) + +// PromotedToActiveEmitter returns a PromotionHooks.EmitPromotedEvent that +// records the promotion against the Lease the new Active just acquired. +// +// The Lease is the right object to attach it to: it *is* the leadership record, +// there is exactly one, and it lives in the controller's own namespace — so the +// Event lands beside the controller that emitted it, discoverable with a plain +// `kubectl get events -n `. +// +// Not kubeslice-system. Issue #297 asks for the Event on that namespace, but +// it does not exist on a hub cluster: per ADR #293 Decision 1 it is a *worker* +// namespace (worker-operator, NSM, gateways, DNS), and the hub's is +// kubeslice-controller / $KUBESLICE_CONTROLLER_MANAGER_NAMESPACE. The trap is +// that a constant with exactly the wrong meaning is sitting in the vendor tree +// (kubeslice-monitoring's logger.ControlPlaneNamespace) waiting to be reached +// for. The recorder derives the Event's namespace from the involved object, so +// passing the Lease is what puts it in the right place. +// +// recorder.RecordEvent is called directly, never util.RecordEvent. That +// helper's first statement is util.CtxLogger(ctx), which nil-panics on any +// context that has not been through PrepareKubeSliceControllersRequestContext — +// and promotion runs on main.go's signal-handler context, which has not. This +// crashed a live Standby once during #295; the fix is not to reach for the +// helper out of habit. +func PromotedToActiveEmitter(recorder events.EventRecorder) func(context.Context, *coordinationv1.Lease) error { + if recorder == nil { + return nil + } + return func(ctx context.Context, lease *coordinationv1.Lease) error { + if lease == nil { + // Nothing to attach the Event to. Promotion has already succeeded by + // this point, so this is worth reporting but not worth failing over. + return nil + } + return recorder.RecordEvent(ctx, &events.Event{ + Object: lease, + ReportingInstance: util.InstanceController, + Name: ossEvents.EventHAPromotedToActive, + }) + } +} diff --git a/pkg/ha/promotion_event_test.go b/pkg/ha/promotion_event_test.go new file mode 100644 index 000000000..8e8431e56 --- /dev/null +++ b/pkg/ha/promotion_event_test.go @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "testing" + "time" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" +) + +func promotedEvents(t *testing.T, c client.Client, namespace string) []corev1.Event { + t.Helper() + list := &corev1.EventList{} + require.NoError(t, c.List(context.Background(), list, client.InNamespace(namespace))) + var out []corev1.Event + for _, ev := range list.Items { + if ev.Reason == "PromotedToActive" { + out = append(out, ev) + } + } + return out +} + +// TestPromotedToActiveEvent_LandsInTheControllersOwnNamespace is the reason the +// Lease is the involved object. Issue #297 asks for this Event on +// kubeslice-system, which does not exist on a hub — ADR #293 Decision 1 is +// explicit that it is a worker-cluster namespace. The recorder derives the +// Event's namespace from the involved object, so attaching it to the Lease is +// what puts it beside the controller that emitted it. +func TestPromotedToActiveEvent_LandsInTheControllersOwnNamespace(t *testing.T) { + eventsClient := fakeClient(t) + recorder := testEventRecorder(t, eventsClient, ossEvents.EventsMap) + emit := PromotedToActiveEmitter(recorder) + require.NotNil(t, emit) + + lease := newLease(DefaultLeaseName, "kubeslice-controller", "hub-b", time.Now()) + require.NoError(t, emit(context.Background(), lease)) + + got := promotedEvents(t, eventsClient, "kubeslice-controller") + require.Len(t, got, 1, "promotion must surface as a Kubernetes event") + assert.Equal(t, DefaultLeaseName, got[0].InvolvedObject.Name, + "the Lease is the leadership record, so it is what the event describes") + assert.Equal(t, "Lease", got[0].InvolvedObject.Kind) + assert.Equal(t, corev1.EventTypeNormal, got[0].Type, "a successful failover is not a warning") + + // And nothing landed in the worker namespace the issue names. + assert.Empty(t, promotedEvents(t, eventsClient, "kubeslice-system"), + "kubeslice-system is a worker-cluster namespace and does not exist on a hub") +} + +// TestPromotedToActiveEvent_FollowsTheLeaseNamespace: the namespace is not +// hardcoded anywhere, it comes from wherever the Lease actually lives — which +// is the downward-API-derived namespace the controller is deployed into. +func TestPromotedToActiveEvent_FollowsTheLeaseNamespace(t *testing.T) { + eventsClient := fakeClient(t) + emit := PromotedToActiveEmitter(testEventRecorder(t, eventsClient, ossEvents.EventsMap)) + + lease := newLease(DefaultLeaseName, "kubeslice-avesha", "hub-b", time.Now()) + require.NoError(t, emit(context.Background(), lease)) + + assert.Len(t, promotedEvents(t, eventsClient, "kubeslice-avesha"), 1, + "a controller deployed into a non-default namespace must emit there, not into a fixed one") +} + +// TestPromotedToActiveEvent_RequiresGeneratedMapEntry guards the generation +// step. RecordEvent hard-fails on an event name that is not in EventsMap, so a +// hand-written config/events/controller.yaml entry without a `make +// generate-events` run would fail at the worst possible moment — during a real +// failover. +func TestPromotedToActiveEvent_RequiresGeneratedMapEntry(t *testing.T) { + schema, ok := ossEvents.EventsMap[ossEvents.EventHAPromotedToActive] + require.True(t, ok, + "EventHAPromotedToActive is missing from the generated EventsMap — run `make generate-events`") + assert.Equal(t, "PromotedToActive", schema.Reason) + assert.Equal(t, events.EventTypeNormal, schema.Type) + + // And the failure being guarded against is loud rather than silent. + unregistered := testEventRecorder(t, fakeClient(t), map[events.EventName]*events.EventSchema{}) + emit := PromotedToActiveEmitter(unregistered) + assert.Error(t, emit(context.Background(), newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-b", time.Now())), + "an unregistered event name must surface as an error rather than being dropped") +} + +func TestPromotedToActiveEmitter_NilRecorderDisablesEmission(t *testing.T) { + assert.Nil(t, PromotedToActiveEmitter(nil), + "a nil recorder must yield a nil hook, which promote() skips") +} + +func TestPromotedToActiveEmitter_NilLeaseIsNotFatal(t *testing.T) { + emit := PromotedToActiveEmitter(testEventRecorder(t, fakeClient(t), ossEvents.EventsMap)) + assert.NoError(t, emit(context.Background(), nil), + "promotion has already succeeded by this point; a missing lease must not be reported as failure") +} diff --git a/pkg/ha/promotion_guards.go b/pkg/ha/promotion_guards.go new file mode 100644 index 000000000..2b3528276 --- /dev/null +++ b/pkg/ha/promotion_guards.go @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +// The guards answer a different question from the staleness verdict. The +// verdict says "the Active's newest proof of life has aged out". The guards ask +// "is that actually evidence the Active is gone, or evidence of something else?" +// +// They exist because the costs are not symmetric. A missed promotion is +// downtime: visible, and an operator can force a takeover. A false promotion is +// two hubs writing to their own copies of the world at once, silently, with the +// mirror still running in one direction — objects overwriting each other, +// prune's reverse diff resurrecting deletes, workers receiving contradictory +// instructions, and recovery by hand. Two bounded reads to avoid the second is +// a good trade inside a budget that already tolerates leaseDuration + padding. + +// selfHealthy reports whether this hub can reach its own API server. +// +// This is the only cheap way to tell "the Active is gone" apart from "my own +// networking is broken". A dead Active API server, a partition between the +// hubs, and this hub losing its own network all produce byte-identical +// observations from here: reads of the remote Lease simply stop succeeding. +// Asking whether the local API server still answers is what separates the last +// case from the first two — and it is the most likely cause of a false +// promotion after outright misconfiguration. +// +// It reads this hub's own Lease through the existing local client, so it needs +// no new client and no new RBAC: config/rbac/leader_election_role.yaml already +// grants leases in the controller's own namespace. +// +// NotFound counts as HEALTHY, and getting this backwards would block every +// real first failover. On a first-ever promotion no Lease exists on this hub +// yet, and a NotFound response means the API server answered — which is +// precisely the thing being tested. Only transport errors, timeouts and server +// errors indicate an unhealthy self. +func (e *ClusterLeaderElector) selfHealthy(ctx context.Context) bool { + selfCtx, cancel := context.WithTimeout(ctx, e.promotionDialTimeout) + defer cancel() + + _, err := getLease(selfCtx, e.localClient, e.leaseName, e.leaseNS) + if err == nil || apierrors.IsNotFound(err) { + return true + } + e.log.Errorw("refusing to promote: this hub cannot reach its own API server", + "error", err, "timeout", e.promotionDialTimeout) + return false +} + +// activeStillAlive performs the final dial: one fresh, timeout-bounded read of +// the Active's Lease at decision time. It reports true only if that read +// succeeds AND returns a Lease that is not stale — i.e. the Active is +// demonstrably still holding leadership and promotion must be abandoned. +// +// What this buys, honestly, differs per path: +// +// - Active reachable but its renewTime frozen: real value. Polling happens +// every retryPeriod, so the Active could have renewed moments after the +// last poll. One fresh read at decision time closes that race. +// - Active unreachable: almost nothing. It is the next failed read after a +// sustained run of failed reads, and it only catches an outage that ended +// within the last tick. +// +// It is NOT a split-brain guard, and nothing in this codebase should imply +// otherwise. In a genuine sustained partition this read travels the same broken +// path as every other read, fails in the same way, and the Standby promotes +// regardless. What actually provides safety on the unreachable path is +// duration — a sustained failure across the whole leaseDuration + padding +// budget rather than one bad read — and the arming rule. Split-brain remains +// the explicit non-goal of ADR #293 Decision 8. +// +// The bound is not optional. main.go builds the remote client with a plain +// uncached client.New and no timeout, so a dial to a black-holed API server +// (packets dropped, no RST) blocks until the OS TCP timeout — minutes, far +// outside the failover budget. Every call here is wrapped. +func (e *ClusterLeaderElector) activeStillAlive(ctx context.Context) bool { + dialCtx, cancel := context.WithTimeout(ctx, e.promotionDialTimeout) + defer cancel() + + lease, err := getLease(dialCtx, e.remoteClient, e.leaseName, e.leaseNS) + if err != nil { + e.log.Infow("final dial to the active hub failed; treating it as gone", + "error", err, "timeout", e.promotionDialTimeout) + return false + } + if isLeaseStale(lease, e.padding, time.Now()) { + e.log.Infow("final dial reached the active hub but its lease is still stale; proceeding", + "holder", leaseHolder(lease), "renewTime", leaseRenewStr(lease)) + return false + } + + // The Active renewed between our last poll and now. Refresh the cached view + // so the next tick evaluates against this newer evidence rather than + // re-deriving the same stale verdict. + e.lastSeenLease = lease + e.lastGoodRead = time.Now() + e.log.Infow("refusing to promote: the active hub renewed its lease between polls", + "holder", leaseHolder(lease), "renewTime", leaseRenewStr(lease)) + return true +} + +// guardsAllowPromotion runs both guards in order and records why it refused. +// Aborting deliberately changes nothing else: the cached Lease is not cleared +// and the elector is not disarmed, so the next tick re-evaluates from scratch +// and a genuinely recovered Active heals the state naturally on its next +// successful read. +func (e *ClusterLeaderElector) guardsAllowPromotion(ctx context.Context) bool { + if !e.selfHealthy(ctx) { + e.abortPromotion(ctx, abortSelfUnhealthy) + return false + } + if e.activeStillAlive(ctx) { + e.abortPromotion(ctx, abortLeaseLive) + return false + } + return true +} diff --git a/pkg/ha/promotion_guards_test.go b/pkg/ha/promotion_guards_test.go new file mode 100644 index 000000000..74753f803 --- /dev/null +++ b/pkg/ha/promotion_guards_test.go @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +// failingReadClient returns a client whose Get always fails with a transport- +// style error, simulating an API server that is unreachable rather than one +// that answers "not found". +func failingReadClient(t *testing.T) client.Client { + t.Helper() + return fake.NewClientBuilder().WithScheme(testScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + return fmt.Errorf("simulated API server unreachable") + }, + }).Build() +} + +// TestSelfHealthy_NotFoundCountsAsHealthy is the guard's most important test. +// On a first-ever promotion there is no Lease on this hub's own cluster yet, so +// the self-health read returns NotFound — and NotFound means the API server +// ANSWERED, which is exactly the thing being tested. Treating it as unhealthy +// would block every real first failover while looking perfectly reasonable in +// review. +func TestSelfHealthy_NotFoundCountsAsHealthy(t *testing.T) { + // An empty local cluster: the HA Lease does not exist yet. + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + assert.True(t, e.selfHealthy(context.Background()), + "NotFound means the API server answered — on a first-ever promotion no local Lease exists, "+ + "and treating that as unhealthy would block every real first failover") +} + +func TestSelfHealthy_ExistingLeaseIsHealthy(t *testing.T) { + local := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-b", time.Now())) + e := NewClusterLeaderElector(local, fakeClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + assert.True(t, e.selfHealthy(context.Background())) +} + +func TestSelfHealthy_UnreachableIsUnhealthy(t *testing.T) { + e := NewClusterLeaderElector(failingReadClient(t), fakeClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + assert.False(t, e.selfHealthy(context.Background()), + "a transport error against our own API server means we may be the broken one, not the Active") +} + +func TestActiveStillAlive_FreshLeaseAborts(t *testing.T) { + remote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + e := NewClusterLeaderElector(fakeClient(t), remote, Options{Mode: ModeStandby, Log: testLog()}) + + assert.True(t, e.activeStillAlive(context.Background()), + "the Active renewed between polls; this is the polling race the final dial exists to catch") +} + +// TestActiveStillAlive_RefreshesCacheOnAbort: aborting must leave the elector +// better informed, not latched on the stale verdict it just disproved. +func TestActiveStillAlive_RefreshesCacheOnAbort(t *testing.T) { + remote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + e := NewClusterLeaderElector(fakeClient(t), remote, Options{Mode: ModeStandby, Log: testLog()}) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + require.True(t, e.activeStillAlive(context.Background())) + + candidate, err := e.checkRemoteLeaseOnce(context.Background()) + require.NoError(t, err) + assert.False(t, candidate, "the refreshed view must clear candidacy on the next tick") +} + +func TestActiveStillAlive_UnreachableProceeds(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + assert.False(t, e.activeStillAlive(context.Background()), + "an unreachable Active is not evidence of life; the final dial buys almost nothing on this path") +} + +func TestActiveStillAlive_StaleLeaseProceeds(t *testing.T) { + remote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour))) + e := NewClusterLeaderElector(fakeClient(t), remote, Options{Mode: ModeStandby, Log: testLog()}) + + assert.False(t, e.activeStillAlive(context.Background()), + "reachable but still stale confirms the verdict rather than refuting it") +} + +func TestGuardsAllowPromotion_BothPass(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + assert.True(t, e.guardsAllowPromotion(context.Background()), + "own API server reachable (NotFound) and Active unreachable: promotion may proceed") +} + +func TestGuardsAllowPromotion_SelfUnhealthyBlocks(t *testing.T) { + // Both sides unreachable — the classic "it's me, not them" case. + e := NewClusterLeaderElector(failingReadClient(t), failingReadClient(t), Options{Mode: ModeStandby, Log: testLog()}) + + assert.False(t, e.guardsAllowPromotion(context.Background()), + "if this hub cannot reach its own API server, the evidence is equally consistent with "+ + "this hub being the broken one") +} + +func TestGuardsAllowPromotion_LiveActiveBlocks(t *testing.T) { + remote := fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + e := NewClusterLeaderElector(fakeClient(t), remote, Options{Mode: ModeStandby, Log: testLog()}) + + assert.False(t, e.guardsAllowPromotion(context.Background())) +} + +// TestGuardsAbort_DoesNotDisarm: an abort must change nothing except the +// promotion attempt itself. Clearing the cached Lease would silently disarm the +// elector, and an already-gone Active can never re-arm it — turning one +// transient guard failure into a hub that will never fail over again. +func TestGuardsAbort_DoesNotDisarm(t *testing.T) { + e := NewClusterLeaderElector(failingReadClient(t), failingReadClient(t), Options{Mode: ModeStandby, Log: testLog()}) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + require.False(t, e.guardsAllowPromotion(context.Background())) + + assert.NotNil(t, e.lastSeenLease, "aborting must not clear the cached lease") + candidate, _ := e.checkRemoteLeaseOnce(context.Background()) + assert.True(t, candidate, "the next tick must re-evaluate and still see a candidate") +} + +// TestPromotionDialTimeout_IsApplied proves the bound exists at all. main.go +// builds the remote client with no timeout, so an unbounded guard would hang +// for the OS TCP timeout — minutes, far outside the failover budget. +func TestPromotionDialTimeout_IsApplied(t *testing.T) { + blocked := make(chan struct{}) + defer close(blocked) + + hanging := fake.NewClientBuilder().WithScheme(testScheme(t)).WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-blocked: + return nil + } + }, + }).Build() + + e := NewClusterLeaderElector(hanging, hanging, Options{ + Mode: ModeStandby, + PromotionDialTimeout: 50 * time.Millisecond, + Log: testLog(), + }) + + done := make(chan bool, 1) + go func() { done <- e.selfHealthy(context.Background()) }() + + select { + case healthy := <-done: + assert.False(t, healthy, "a timed-out read is not proof of a healthy self") + case <-time.After(2 * time.Second): + t.Fatal("selfHealthy did not respect PromotionDialTimeout — an unbounded read would " + + "hang until the OS TCP timeout") + } +} diff --git a/pkg/ha/promotion_test.go b/pkg/ha/promotion_test.go new file mode 100644 index 000000000..8b5cf36b0 --- /dev/null +++ b/pkg/ha/promotion_test.go @@ -0,0 +1,710 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + coordinationv1 "k8s.io/api/coordination/v1" +) + +// promotionRecorder records the order in which hooks ran and what the write +// fence reported at each point. Ordering is the whole subject of several tests +// below, so it is captured rather than inferred. +type promotionRecorder struct { + mu sync.Mutex + steps []string + fenceOpenAt map[string]bool + stopMirror error + publishErr error + kickErr error + publishDelay time.Duration +} + +func newPromotionRecorder() *promotionRecorder { + return &promotionRecorder{fenceOpenAt: map[string]bool{}} +} + +func (r *promotionRecorder) record(step string, e *ClusterLeaderElector) { + r.mu.Lock() + defer r.mu.Unlock() + r.steps = append(r.steps, step) + r.fenceOpenAt[step] = e.IsLeader() +} + +func (r *promotionRecorder) order() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.steps...) +} + +func (r *promotionRecorder) hooks(e *ClusterLeaderElector) PromotionHooks { + return PromotionHooks{ + StopMirror: func(ctx context.Context) error { + r.record("stopMirror", e) + return r.stopMirror + }, + PublishActiveController: func(ctx context.Context) error { + r.record("publish", e) + if r.publishDelay > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(r.publishDelay): + } + } + return r.publishErr + }, + KickReconcilers: func(ctx context.Context) error { + r.record("kick", e) + return r.kickErr + }, + EmitPromotedEvent: func(ctx context.Context, lease *coordinationv1.Lease) error { + r.record("event", e) + return nil + }, + } +} + +// standbyReadyToPromote builds a Standby whose Active is unreachable and whose +// cached view has aged out — i.e. one tick away from promoting. +func standbyReadyToPromote(t *testing.T) *ClusterLeaderElector { + t.Helper() + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + return e +} + +func TestPromote_HappyPath(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.True(t, promoted) + + assert.True(t, e.IsLeader(), "the write fence must be open once promotion completes") + assert.Equal(t, ModeActive, e.Mode()) + + lease, err := getLease(context.Background(), e.localClient, e.leaseName, e.leaseNS) + require.NoError(t, err, "promotion must acquire the lease on this hub's own cluster") + assert.Equal(t, "hub-b", leaseHolder(lease)) +} + +// TestPromote_StopsMirrorBeforeOpeningTheFence is the ordering bug that both +// issue #297's own sequence gets wrong. In the most +// common trigger — the Active's pod dies while its API server stays healthy — +// the mirror is still live at this moment. Because every mirrored object +// carries the syncer's label and the mirror's conflict guard only skips objects +// WITHOUT it, a fence opened before the mirror stopped means the mirror +// overwrites exactly what the new Active writes, and prune's reverse diff +// resurrects whatever it deletes. +func TestPromote_StopsMirrorBeforeOpeningTheFence(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.True(t, promoted) + + order := rec.order() + require.Contains(t, order, "stopMirror") + assert.Equal(t, "stopMirror", order[0], "the mirror must be stopped first, before anything else") + assert.False(t, rec.fenceOpenAt["stopMirror"], + "the write fence must still be SHUT while the mirror is being stopped — otherwise the "+ + "promoted hub and the still-running mirror write to the same objects") + assert.False(t, rec.fenceOpenAt["publish"], + "the fence must still be shut while activeController is published") +} + +// TestPromote_FenceOpensOnlyAfterPublish covers the other half: the kick and +// the event must land on a hub whose fence is already open, or the kick's +// re-enqueued requests are dropped without requeue — the exact failure the kick +// exists to fix. +func TestPromote_FenceOpensOnlyAfterPublish(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + _, err := e.promote(context.Background()) + require.NoError(t, err) + + assert.Equal(t, []string{"stopMirror", "publish", "kick", "event"}, rec.order(), + "promotion order is load-bearing, not incidental") + assert.True(t, rec.fenceOpenAt["kick"], + "the kick must run with the fence OPEN, or every re-enqueued request is dropped without requeue") + assert.True(t, rec.fenceOpenAt["event"]) +} + +// TestPromote_FenceStaysShutThroughout is the property steps 0 and 8 exist to +// provide, checked from outside the hooks: at no point during the sequence may +// a concurrent Reconcile see itself as leader. +func TestPromote_FenceStaysShutThroughout(t *testing.T) { + e := standbyReadyToPromote(t) + // Pretend a previous life left isLeader set. Only the promoting latch should + // be keeping the fence shut. + e.isLeader.Store(true) + + observed := make(chan bool, 64) + rec := newPromotionRecorder() + hooks := rec.hooks(e) + inner := hooks.StopMirror + hooks.StopMirror = func(ctx context.Context) error { + observed <- e.IsLeader() + return inner(ctx) + } + e.SetPromotionHooks(hooks) + + _, err := e.promote(context.Background()) + require.NoError(t, err) + close(observed) + + for sawLeader := range observed { + assert.False(t, sawLeader, + "IsLeader() must report false for the whole sequence even when isLeader is set, "+ + "because the promoting latch overrides it") + } + assert.True(t, e.IsLeader(), "and must report true once the sequence completes") +} + +// TestPromote_GuardRefusalIsNotAnError: declining to promote is a correct +// outcome. It must leave the hub a fenced, still-armed Standby. +func TestPromote_GuardRefusalLeavesHubUnchanged(t *testing.T) { + // Self-health fails: this hub cannot reach its own API server. + e := NewClusterLeaderElector(failingReadClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err, "a guard refusal is a correct outcome, not an error") + assert.False(t, promoted) + + assert.False(t, e.IsLeader(), "still fenced") + assert.Equal(t, ModeStandby, e.Mode(), "still a standby") + assert.False(t, e.promoting.Load(), "the promotion latch must be released so the next tick can retry") + assert.NotNil(t, e.lastSeenLease, "still armed") + assert.Empty(t, rec.order(), "no hook may run once a guard has refused") +} + +// TestPromote_MirrorStopFailureAborts: a half-stopped mirror is exactly the +// dual-writer state step 3 exists to prevent, so failing to stop it must abort +// the promotion rather than press on. +func TestPromote_MirrorStopFailureAborts(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + rec.stopMirror = fmt.Errorf("simulated: syncer did not stop") + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.Error(t, err, "the failure must surface, not be swallowed") + assert.False(t, promoted) + + assert.False(t, e.IsLeader(), "the fence must stay shut") + assert.Equal(t, ModeStandby, e.Mode()) + assert.False(t, e.promoting.Load(), "the latch must be released so the next tick can retry") + assert.Equal(t, []string{"stopMirror"}, rec.order(), "nothing after the mirror stop may run") +} + +// TestPromote_AbortAfterMirrorStopIsLoud pins the one exit path that does NOT +// restore the hub to exactly its prior state. Stopping the mirror is one-way — +// nothing in this package can start it again — so a promotion that aborts after +// step 3 leaves a Standby that no longer mirrors. That is recoverable on a later +// promotion but permanent if the Active recovers instead, and the only thing +// standing between an operator and a silently diverging Standby is this log +// line. If the message moves, the failure mode goes back to being invisible. +func TestPromote_AbortAfterMirrorStopIsLoud(t *testing.T) { + core, logs := observer.New(zapcore.ErrorLevel) + + e := standbyReadyToPromote(t) + e.log = zap.New(core).Sugar() + rec := newPromotionRecorder() + rec.stopMirror = fmt.Errorf("simulated: syncer did not stop") + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.Error(t, err) + require.False(t, promoted) + + assert.Equal(t, 1, logs.FilterMessageSnippet("no longer mirroring").Len(), + "aborting after the mirror was stopped must be reported at error level") +} + +// TestPromote_AbortBeforeMirrorStopIsQuiet is the other half: a guard refusal +// happens before step 3, the mirror is untouched, and warning about a dead +// mirror there would be noise on the most ordinary abort there is. +func TestPromote_AbortBeforeMirrorStopIsQuiet(t *testing.T) { + core, logs := observer.New(zapcore.ErrorLevel) + + e := standbyReadyToPromote(t) + e.log = zap.New(core).Sugar() + // A live Active: the final-dial guard refuses before the mirror is stopped. + e.remoteClient = fakeClient(t, newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now())) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.False(t, promoted) + + assert.Empty(t, rec.order(), "no hook may run once a guard has refused") + assert.Equal(t, 0, logs.FilterMessageSnippet("no longer mirroring").Len(), + "the mirror was never stopped, so nothing may claim it was") +} + +// TestPromote_PublishFailureStillPromotes: publication is a budget, not a +// precondition. A hub that cannot describe itself is still a better Active than +// no Active at all, and the publisher's own loop keeps retrying. +func TestPromote_PublishFailureStillPromotes(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + rec.publishErr = fmt.Errorf("simulated: API server busy") + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + assert.True(t, promoted, "failing to publish must not strand the cluster with no writer") + assert.True(t, e.IsLeader()) + assert.Contains(t, rec.order(), "kick", "the rest of the sequence must still run") +} + +// TestPromote_PublishRespectsGracePeriod: a publication that hangs must not +// hold the write fence shut indefinitely. +func TestPromote_PublishRespectsGracePeriod(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + PromotionGracePeriod: 50 * time.Millisecond, + Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + rec := newPromotionRecorder() + rec.publishDelay = 10 * time.Second + e.SetPromotionHooks(rec.hooks(e)) + + done := make(chan struct{}) + go func() { + defer close(done) + promoted, err := e.promote(context.Background()) + assert.NoError(t, err) + assert.True(t, promoted) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("promotion did not bound the publish step by PromotionGracePeriod — a hung " + + "publication must not hold the write fence shut indefinitely") + } + assert.True(t, e.IsLeader()) +} + +// TestPromote_KickFailureStillPromotes: the kick is important enough to log +// loudly about and not important enough to abandon a promotion for. Without it +// pre-existing mirrored state stays unreconciled, but the hub is still a +// working Active for anything new. +func TestPromote_KickFailureStillPromotes(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + rec.kickErr = fmt.Errorf("simulated: channel full") + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + assert.True(t, promoted) + assert.Contains(t, rec.order(), "event", "the promotion must still be recorded") +} + +func TestPromote_IsOnceOnly(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + require.True(t, promoted) + + // A second attempt on an already-promoted hub must be a no-op. Re-running + // the sequence would race the renewal loop promotion just started — that + // goroutine owns lastRenew and is actively writing the Lease — and fight it + // for the same object. -race found this during development, and this test is + // what keeps it found. + before := len(rec.order()) + promoted2, err := e.promote(context.Background()) + require.NoError(t, err) + assert.True(t, promoted2, "an already-active hub is already in the state promotion produces") + assert.Equal(t, before, len(rec.order()), + "no hook may run a second time — the sequence must be genuinely once-only, not merely idempotent") + assert.True(t, e.IsLeader()) +} + +func TestPromote_NilHooksAreSkipped(t *testing.T) { + e := standbyReadyToPromote(t) + // No SetPromotionHooks call at all. + + promoted, err := e.promote(context.Background()) + require.NoError(t, err, "an elector with no hooks must still be able to take leadership") + assert.True(t, promoted) + assert.True(t, e.IsLeader()) + assert.Equal(t, ModeActive, e.Mode()) +} + +// TestWatchRemoteLease_PromotesAndReturns is the end-to-end path: a Standby +// watching a dead Active must promote and stop watching, because there is no +// longer an Active to watch. +func TestWatchRemoteLease_PromotesAndReturns(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + RetryPeriod: 10 * time.Millisecond, + Log: testLog(), + }) + // Armed against a Lease that has already aged out. + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + errCh := make(chan error, 1) + go func() { errCh <- e.WatchRemoteLease(ctx) }() + + select { + case err := <-errCh: + require.NoError(t, err) + assert.True(t, e.IsLeader(), "the watch must have promoted before returning") + assert.Equal(t, ModeActive, e.Mode()) + case <-time.After(3 * time.Second): + t.Fatal("WatchRemoteLease did not promote and return") + } +} + +// TestWatchRemoteLease_NeverArmedNeverPromotes is the same loop under the +// failure that matters most: a Standby that has never read the Active's Lease +// must sit there warning forever rather than promoting itself. +func TestWatchRemoteLease_NeverArmedNeverPromotes(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), fakeClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + RetryPeriod: 5 * time.Millisecond, + Log: testLog(), + }) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + require.NoError(t, e.WatchRemoteLease(ctx)) + + assert.False(t, e.IsLeader(), "an unarmed standby must never promote, however many ticks pass") + assert.Equal(t, ModeStandby, e.Mode()) + assert.Empty(t, rec.order(), "the promotion sequence must never have started") +} + +// TestPromote_ConcurrentAttemptsRunTheSequenceOnce exercises the latch under +// real concurrency rather than trusting the comment on it. Two goroutines enter +// promote at the same time; exactly one may run the sequence, and neither may +// see a half-promoted hub. +func TestPromote_ConcurrentAttemptsRunTheSequenceOnce(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + + // Hold the sequence open inside the first hook so the second caller is + // guaranteed to arrive while the first is still mid-flight. + release := make(chan struct{}) + entered := make(chan struct{}, 1) + hooks := rec.hooks(e) + inner := hooks.StopMirror + hooks.StopMirror = func(ctx context.Context) error { + select { + case entered <- struct{}{}: + default: + } + <-release + return inner(ctx) + } + e.SetPromotionHooks(hooks) + + results := make(chan bool, 2) + errs := make(chan error, 2) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ok, err := e.promote(context.Background()) + results <- ok + errs <- err + }() + + <-entered // the first caller is inside the sequence, holding the latch + + wg.Add(1) + go func() { + defer wg.Done() + ok, err := e.promote(context.Background()) + results <- ok + errs <- err + }() + // Give the second caller time to hit the latch and bail before releasing. + time.Sleep(50 * time.Millisecond) + close(release) + wg.Wait() + close(results) + close(errs) + + for err := range errs { + require.NoError(t, err, "a rejected concurrent attempt is not an error") + } + promotedCount := 0 + for ok := range results { + if ok { + promotedCount++ + } + } + assert.Equal(t, 1, promotedCount, + "exactly one of two concurrent attempts may report having promoted") + assert.Equal(t, []string{"stopMirror", "publish", "kick", "event"}, rec.order(), + "the sequence must have run exactly once, not twice and not partially") + assert.True(t, e.IsLeader()) +} + +// TestPromote_LeaseAcquisitionFailureAborts covers the one step whose failure +// means the hub genuinely cannot lead: without the Lease there is nothing +// fencing the old Active, so opening the write fence anyway would be the +// dual-writer state the whole design exists to prevent. +func TestPromote_LeaseAcquisitionFailureAborts(t *testing.T) { + // Reads succeed (so self-health passes) but writes fail, so the Lease + // cannot be created. + e := NewClusterLeaderElector(failingWriteClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + rec := newPromotionRecorder() + e.SetPromotionHooks(rec.hooks(e)) + + promoted, err := e.promote(context.Background()) + require.Error(t, err, "failing to take the lease must surface") + assert.False(t, promoted) + + assert.False(t, e.IsLeader(), "the write fence must stay shut without a lease") + assert.Equal(t, ModeStandby, e.Mode(), "mode must not have flipped") + assert.False(t, e.promoting.Load(), "the latch must be released so the next tick can retry") + assert.Equal(t, []string{"stopMirror"}, rec.order(), + "nothing past the lease acquisition may run") +} + +// TestPromote_EventFailureStillPromotes: the Event is a report, not a step. A +// hub that took over but could not say so is still the Active. +func TestPromote_EventFailureStillPromotes(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + hooks := rec.hooks(e) + hooks.EmitPromotedEvent = func(ctx context.Context, lease *coordinationv1.Lease) error { + rec.record("event", e) + return fmt.Errorf("simulated: event recorder unavailable") + } + e.SetPromotionHooks(hooks) + + promoted, err := e.promote(context.Background()) + require.NoError(t, err) + assert.True(t, promoted, "failing to report a promotion must not undo it") + assert.True(t, e.IsLeader()) +} + +// TestPromote_AttachesTheAcquiredLeaseToTheEvent: the Event must describe the +// Lease this hub just took, not some other object — that is what puts it in the +// controller's own namespace. +func TestPromote_AttachesTheAcquiredLeaseToTheEvent(t *testing.T) { + e := standbyReadyToPromote(t) + rec := newPromotionRecorder() + hooks := rec.hooks(e) + var got *coordinationv1.Lease + hooks.EmitPromotedEvent = func(ctx context.Context, lease *coordinationv1.Lease) error { + got = lease + return nil + } + e.SetPromotionHooks(hooks) + + _, err := e.promote(context.Background()) + require.NoError(t, err) + + require.NotNil(t, got, "the event hook must receive the acquired lease") + assert.Equal(t, DefaultLeaseName, got.Name) + assert.Equal(t, "hub-b", leaseHolder(got), "the lease must already name this hub as holder") +} + +// TestPromote_BoundsTheMirrorStop is a regression test for a gap found by +// auditing the failure paths rather than the happy one. +// +// Waiting indefinitely for the mirror to stop looks like the safe choice, since +// proceeding without it is the dual-writer state step 3 exists to prevent. It +// is not. The watch loop calls promote synchronously, so an unbounded wait on a +// mirror that never exits blocks the loop: no further polls, no further +// staleness evaluation, no failover ever, and nothing logged after the sequence +// started. "Never promote into a dual writer" silently becomes "never promote", +// which is strictly worse — and invisible. +// +// Bounded, expiry aborts loudly and the next tick retries. +func TestPromote_BoundsTheMirrorStop(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + PromotionGracePeriod: 50 * time.Millisecond, + Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + stopCalled := make(chan struct{}, 1) + e.SetPromotionHooks(PromotionHooks{ + // Exactly what main.go's hook does when the syncer never exits: wait for + // the mirror to confirm, or for the context to give up. + StopMirror: func(ctx context.Context) error { + stopCalled <- struct{}{} + <-ctx.Done() + return ctx.Err() + }, + }) + + done := make(chan struct{}) + var promoted bool + var err error + go func() { + defer close(done) + promoted, err = e.promote(context.Background()) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("promote blocked on a mirror that never stopped — an unbounded wait here " + + "silently disables failover entirely, which is worse than the dual-writer state it avoids") + } + + require.Len(t, stopCalled, 1, "the mirror stop must actually have been attempted") + require.Error(t, err, "a mirror that never confirms must abort the promotion loudly") + assert.False(t, promoted) + + // And the hub must be left able to try again. + assert.False(t, e.IsLeader(), "the fence must stay shut") + assert.Equal(t, ModeStandby, e.Mode()) + assert.False(t, e.promoting.Load(), "the latch must be released so the next tick can retry") + assert.NotNil(t, e.lastSeenLease, "and the hub must stay armed") +} + +// TestPromote_RefusesWithoutARemoteClient states a precondition that is +// otherwise implicit. Without a client to the Active there is no way to have +// observed it alive, so nothing could justify concluding it is gone. It is +// unreachable from the watch loop — which refuses to start without one, and +// could not arm without one either — but promote is a method, and before this +// guard a caller reaching it that way crashed inside the final dial rather than +// being told no. +func TestPromote_RefusesWithoutARemoteClient(t *testing.T) { + e := NewClusterLeaderElector(fakeClient(t), nil, Options{ + Mode: ModeStandby, Identity: "hub-b", Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + + promoted, err := e.promote(context.Background()) + require.Error(t, err, "the precondition must be reported, not hit as a nil dereference") + assert.False(t, promoted) + assert.False(t, e.IsLeader()) + assert.False(t, e.promoting.Load(), "the latch must still be released") +} + +// TestPromote_BoundsThePostFenceHooks covers the two steps that run after +// leadership has already been taken. A hang there cannot cost the failover — +// the hub is Active and writing by then — but it can cost the promotion ever +// finishing or reporting itself, and it leaks the watch goroutine. +// +// The kick is the one that matters in practice: it pushes into a channel per +// reconciled type, and those are only drained once the manager is running, +// while main.go starts the watch loop before mgr.Start. A kick landing in that +// window has nothing reading the other end. +func TestPromote_BoundsThePostFenceHooks(t *testing.T) { + for _, tc := range []struct { + name string + build func(chan struct{}) PromotionHooks + }{ + {"kick", func(block chan struct{}) PromotionHooks { + return PromotionHooks{KickReconcilers: func(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-block: + return nil + } + }} + }}, + {"event", func(block chan struct{}) PromotionHooks { + return PromotionHooks{EmitPromotedEvent: func(ctx context.Context, l *coordinationv1.Lease) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-block: + return nil + } + }} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + block := make(chan struct{}) + defer close(block) + + e := NewClusterLeaderElector(fakeClient(t), failingReadClient(t), Options{ + Mode: ModeStandby, + Identity: "hub-b", + PromotionGracePeriod: 50 * time.Millisecond, + Log: testLog(), + }) + e.lastSeenLease = newLease(DefaultLeaseName, DefaultLeaseNamespace, "hub-a", time.Now().Add(-time.Hour)) + e.SetPromotionHooks(tc.build(block)) + + done := make(chan struct{}) + var promoted bool + go func() { + defer close(done) + promoted, _ = e.promote(context.Background()) + }() + + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatalf("promote blocked on a hung %s hook; the fence is already open, so this "+ + "leaves a promoted hub whose promotion never finishes or reports itself", tc.name) + } + assert.True(t, promoted, "a hung post-fence hook must not undo a completed takeover") + assert.True(t, e.IsLeader()) + }) + } +} diff --git a/pkg/ha/prune.go b/pkg/ha/prune.go new file mode 100644 index 000000000..dcdea0980 --- /dev/null +++ b/pkg/ha/prune.go @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// DefaultPruneInterval is how often the prune backstop diffs the Standby's +// mirrored objects against the Active hub. Overridable through +// RemoteSyncerOptions.PruneInterval (--ha-sync-interval in main.go). +const DefaultPruneInterval = 60 * time.Second + +// opPrune labels ha_sync_errors_total increments coming from the prune +// loop's own list calls; failures of the deletes it triggers are labeled by +// the worker that performs them (opDelete), like any other queued item. +const opPrune mirrorOp = "prune" + +// remoteListFunc lists, for one GVK, the keys of every object currently +// present on the Active hub. The real implementation (listFromRemoteCache) +// reads controller-runtime's cache — a local-indexer read, not a network +// call. Overridable in tests, the same seam pattern as remoteGetFunc. +type remoteListFunc func(ctx context.Context, gvk schema.GroupVersionKind) (map[syncKey]struct{}, error) + +// runPrune periodically reconciles drift between the Standby's mirrors and +// the Active hub that the informers never reported. Forward direction: a +// mirrored object whose Active-side original was deleted while this process +// wasn't watching (e.g. between two Standby runs) never gets a Delete event +// — cold-start informers only deliver what currently exists — so its mirror +// would otherwise survive as an orphan forever. Reverse direction: an +// Active-side object with no Standby mirror is re-enqueued (see pruneOnce +// for the cases that produces). The workqueue still owns +// retry-on-transient-failure; this loop only feeds it. +// +// It blocks until the remote cache has synced before the first pass: an +// unsynced cache lists empty, and an empty "Active" view would read as +// "everything was deleted" and prune every mirror on the Standby. +func (s *RemoteSyncer) runPrune(ctx context.Context) { + if !s.waitForCacheSync(ctx) { + return // ctx cancelled before the remote cache ever synced + } + ticker := time.NewTicker(s.pruneInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.pruneOnce(ctx) + } + } +} + +// pruneOnce diffs one round: for each mirrored type, every Standby object +// carrying the sync label but no longer present on the Active hub is +// enqueued onto the ordinary mirror workqueue rather than deleted here +// directly — the worker re-reads Active at dequeue time (so an object that +// reappeared in the meantime is mirrored, not deleted), and the conflict +// guard and rate-limited retry apply unchanged, with no second write path +// racing the workers. +// +// A failed list skips that type for this round instead of pruning on partial +// information — deleting mirrors is the one operation where acting on an +// incomplete view is worse than doing nothing until the next tick. +func (s *RemoteSyncer) pruneOnce(ctx context.Context) { + for _, res := range s.resources { + activeKeys, err := s.remoteList(ctx, res.GVK) + if err != nil { + haSyncErrorsTotal.WithLabelValues(res.GVK.Kind, string(opPrune)).Inc() + s.log.Warnw("prune: listing active hub failed; skipping kind this round", + "kind", res.GVK.Kind, "error", err) + continue + } + + local := &unstructured.UnstructuredList{} + local.SetGroupVersionKind(res.GVK.GroupVersion().WithKind(res.GVK.Kind + "List")) + if err := s.localClient.List(ctx, local, + client.MatchingLabels{LabelSyncedFromActive: LabelValueActive}); err != nil { + haSyncErrorsTotal.WithLabelValues(res.GVK.Kind, string(opPrune)).Inc() + s.log.Warnw("prune: listing local mirrors failed; skipping kind this round", + "kind", res.GVK.Kind, "error", err) + continue + } + + localKeys := make(map[syncKey]struct{}, len(local.Items)) + for i := range local.Items { + item := &local.Items[i] + key := syncKey{GVK: res.GVK, Namespace: item.GetNamespace(), Name: item.GetName()} + localKeys[key] = struct{}{} + if _, onActive := activeKeys[key]; onActive { + continue + } + s.log.Infow("prune: enqueuing orphaned mirror", + "kind", key.GVK.Kind, "namespace", key.Namespace, "name", key.Name) + s.enqueue(key) + } + + // Reverse diff: an Active-side object with no mirror on the Standby. + // Usually a create this loop's forward pass can't see — a mirror + // someone deleted directly on the Standby, an object whose skip + // verdict was decided before its namespace had synced (see + // namespaceIsMirrored), or a key stuck deep in retry backoff. + // Re-enqueueing is always safe: the worker re-reads Active and runs + // the full Skip/namespace/conflict-guard chain, so objects that + // should not mirror simply no-op again. + for key := range activeKeys { + if _, mirrored := localKeys[key]; !mirrored { + haPruneResurrectedTotal.WithLabelValues(key.GVK.Kind).Inc() + s.enqueue(key) + } + } + } + // After the loop, not inside it: one pass covers every kind, and a per-kind + // timestamp would report the last kind processed rather than the last complete + // pass. Set even when some kinds were skipped on a failed list — the pass did + // run, and the skips are already counted on ha_sync_errors_total. + haPruneLastRunTimestamp.WithLabelValues(string(s.mode)).Set(float64(time.Now().Unix())) +} + +// listFromRemoteCache is remoteListFunc's real implementation: a List against +// controller-runtime's cache, served from the informer's local indexer. For +// Namespace this inherits the cache's label scoping (see +// namespaceMirrorSelector), so a namespace that loses the controller label on +// Active disappears from this view exactly as it does from the informer's. +func (s *RemoteSyncer) listFromRemoteCache(ctx context.Context, gvk schema.GroupVersionKind) (map[syncKey]struct{}, error) { + ul := &unstructured.UnstructuredList{} + ul.SetGroupVersionKind(gvk.GroupVersion().WithKind(gvk.Kind + "List")) + if err := s.remoteCache.List(ctx, ul); err != nil { + return nil, err + } + keys := make(map[syncKey]struct{}, len(ul.Items)) + for i := range ul.Items { + item := &ul.Items[i] + keys[syncKey{GVK: gvk, Namespace: item.GetNamespace(), Name: item.GetName()}] = struct{}{} + } + return keys, nil +} diff --git a/pkg/ha/prune_test.go b/pkg/ha/prune_test.go new file mode 100644 index 000000000..2ec6e0880 --- /dev/null +++ b/pkg/ha/prune_test.go @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +// stubRemoteList returns a fixed key set (or error) regardless of GVK, +// standing in for listFromRemoteCache the same way stubRemote stands in for +// getFromRemoteCache. +func stubRemoteList(keys []syncKey, err error) remoteListFunc { + return func(_ context.Context, gvk schema.GroupVersionKind) (map[syncKey]struct{}, error) { + if err != nil { + return nil, err + } + set := make(map[syncKey]struct{}, len(keys)) + for _, k := range keys { + if k.GVK == gvk { + set[k] = struct{}{} + } + } + return set, nil + } +} + +func labeledMirror(gvk schema.GroupVersionKind, namespace, name string) *unstructured.Unstructured { + u := newTestUnstructured(gvk, namespace, name) + u.SetLabels(map[string]string{LabelSyncedFromActive: LabelValueActive}) + return u +} + +func TestPruneOnce_EnqueuesOnlyOrphanedMirrors(t *testing.T) { + ctx := context.Background() + kept := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-kept"} + orphan := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-orphan"} + + s := buildSyncer(t, newStubRemote()) + require.NoError(t, s.localClient.Create(ctx, labeledMirror(testGVK, kept.Namespace, kept.Name))) + require.NoError(t, s.localClient.Create(ctx, labeledMirror(testGVK, orphan.Namespace, orphan.Name))) + s.remoteList = stubRemoteList([]syncKey{kept}, nil) + + s.pruneOnce(ctx) + + require.Equal(t, 1, s.queue.Len(), "only the mirror missing from Active should be enqueued") + got, _ := s.queue.Get() + assert.Equal(t, orphan, got) +} + +func TestPruneOnce_ThenWorkerDeletesOrphan(t *testing.T) { + ctx := context.Background() + orphan := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-orphan"} + + // The stub remote holds nothing, so the worker's re-read reports NotFound + // — the same path an informer-delivered delete takes. + s := buildSyncer(t, newStubRemote()) + require.NoError(t, s.localClient.Create(ctx, labeledMirror(testGVK, orphan.Namespace, orphan.Name))) + s.remoteList = stubRemoteList(nil, nil) + + s.pruneOnce(ctx) + key, shutdown := s.queue.Get() + require.False(t, shutdown) + s.processOnce(ctx, key) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(testGVK) + err := s.localClient.Get(ctx, types.NamespacedName{Namespace: orphan.Namespace, Name: orphan.Name}, got) + assert.True(t, apierrors.IsNotFound(err), "the orphaned mirror should be gone after the worker processes the pruned key") +} + +func TestPruneOnce_LeavesUnlabeledObjectsAlone(t *testing.T) { + ctx := context.Background() + s := buildSyncer(t, newStubRemote()) + // An object the Standby's own users created — no sync label — must never + // be pruned, even though Active has no such object. + require.NoError(t, s.localClient.Create(ctx, newTestUnstructured(testGVK, "proj-a", "hand-created"))) + s.remoteList = stubRemoteList(nil, nil) + + s.pruneOnce(ctx) + + assert.Equal(t, 0, s.queue.Len(), "objects without the sync label are not the engine's to prune") +} + +func TestPruneOnce_SkipsKindWhenRemoteListFails(t *testing.T) { + ctx := context.Background() + orphan := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-orphan"} + + s := buildSyncer(t, newStubRemote()) + require.NoError(t, s.localClient.Create(ctx, labeledMirror(testGVK, orphan.Namespace, orphan.Name))) + s.remoteList = stubRemoteList(nil, fmt.Errorf("simulated transient list failure")) + + s.pruneOnce(ctx) + + assert.Equal(t, 0, s.queue.Len(), "a failed list must not be read as \"everything was deleted on Active\"") +} + +func TestPruneOnce_ReverseDiffEnqueuesActiveObjectsMissingLocally(t *testing.T) { + ctx := context.Background() + missing := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-missing"} + + // Active has an object the Standby has no mirror of — a create the + // forward (orphan) pass can't see: a mirror deleted directly on the + // Standby, a skip decided before the namespace informer synced, or a key + // stuck deep in retry backoff. + s := buildSyncer(t, newStubRemote()) + s.remoteList = stubRemoteList([]syncKey{missing}, nil) + + s.pruneOnce(ctx) + + require.Equal(t, 1, s.queue.Len()) + got, _ := s.queue.Get() + assert.Equal(t, missing, got) +} + +func TestPruneOnce_ReverseDiffCannotOverrideConflictGuard(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "hand-created"} + + // The object exists on both sides, but the Standby's copy is not + // syncer-owned (no sync label) — so it is absent from the forward pass's + // labeled listing and the reverse diff re-enqueues it every round. That + // must stay harmless: the worker's conflict guard refuses the write. + remote := newStubRemote() + remote.objects[key] = newTestUnstructured(testGVK, key.Namespace, key.Name) + s := buildSyncer(t, remote) + require.NoError(t, s.localClient.Create(ctx, newTestUnstructured(testGVK, key.Namespace, key.Name))) + s.remoteList = stubRemoteList([]syncKey{key}, nil) + + s.pruneOnce(ctx) + k, shutdown := s.queue.Get() + require.False(t, shutdown) + require.Equal(t, key, k) + s.processOnce(ctx, k) + + got := getUnstructured(t, s.localClient, key) + assert.NotEqual(t, LabelValueActive, got.GetLabels()[LabelSyncedFromActive], + "a hand-created Standby object must never be adopted by the mirror, even via the prune loop") +} + +func TestRunPrune_DoesNotPruneBeforeCacheSync(t *testing.T) { + s := buildSyncer(t, newStubRemote()) + s.pruneInterval = time.Millisecond + s.waitForCacheSync = func(ctx context.Context) bool { return false } // never syncs (ctx cancelled) + + var listCalls int32 + s.remoteList = func(_ context.Context, _ schema.GroupVersionKind) (map[syncKey]struct{}, error) { + atomic.AddInt32(&listCalls, 1) + return nil, nil + } + + done := make(chan struct{}) + go func() { s.runPrune(context.Background()); close(done) }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runPrune did not return after waitForCacheSync reported failure") + } + assert.Equal(t, int32(0), atomic.LoadInt32(&listCalls), + "pruning against an unsynced cache would delete every mirror; runPrune must bail out instead") +} + +func TestRunPrune_TicksAndStopsOnContextCancel(t *testing.T) { + s := buildSyncer(t, newStubRemote()) + s.pruneInterval = 5 * time.Millisecond + s.waitForCacheSync = func(ctx context.Context) bool { return true } + + var listCalls int32 + s.remoteList = func(_ context.Context, _ schema.GroupVersionKind) (map[syncKey]struct{}, error) { + atomic.AddInt32(&listCalls, 1) + return nil, nil + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { s.runPrune(ctx); close(done) }() + + deadline := time.Now().Add(2 * time.Second) + for atomic.LoadInt32(&listCalls) == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + require.Greater(t, atomic.LoadInt32(&listCalls), int32(0), "the prune loop should tick once the cache is synced") + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("runPrune did not return after context cancellation") + } +} diff --git a/pkg/ha/remote_syncer.go b/pkg/ha/remote_syncer.go new file mode 100644 index 000000000..6452e130c --- /dev/null +++ b/pkg/ha/remote_syncer.go @@ -0,0 +1,595 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/kubeslice/kubeslice-monitoring/pkg/events" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + toolscache "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + ossEvents "github.com/kubeslice/kubeslice-controller/events" + "github.com/kubeslice/kubeslice-controller/util" +) + +// Default RemoteSyncer tunables, overridable through RemoteSyncerOptions. +const ( + DefaultSyncWorkers = 4 + DefaultInformerResyncPeriod = 10 * time.Minute + // DefaultInformerSetupRetryPeriod is how long Start waits between retries + // when wiring up the remote informers fails (e.g. the Active hub is + // briefly unreachable, or its RBAC from config/ha hasn't been applied + // yet at the moment the Standby pod starts). + DefaultInformerSetupRetryPeriod = 5 * time.Second +) + +// namespaceMirrorSelector scopes the Namespace informer to only the +// namespaces this controller itself manages — every project namespace +// NamespaceService.ReconcileProjectNamespace creates carries this label (see +// util.LabelsKubeSliceController). Without it, the informer would watch +// every namespace on the Active hub cluster-wide (kube-system, kube-public, +// unrelated infra namespaces), and mirrorDelete (mirror.go) would delete any +// of those incidentally-mirrored namespaces — cascading to delete everything +// inside them on the Standby — the moment the Active hub deletes its copy +// for a completely unrelated reason. +func namespaceMirrorSelector() labels.Selector { + return labels.SelectorFromSet(util.LabelsKubeSliceController) +} + +// mirrorCacheByObject scopes the remote cache's informers server-side, so +// unrelated Active-hub objects never reach this process at all — the mirror +// set's Skip predicates enforce the same boundaries client-side, but for +// core types that exist cluster-wide (Secrets above all) not watching them +// in the first place is both the cheaper and the safer layer. +// +// - Namespace, ServiceAccount, Role, RoleBinding: label-scoped to +// util.LabelsKubeSliceController — stamped on every project namespace by +// ReconcileProjectNamespace and on every credential object the +// controller creates via util.GetOwnerLabel (which embeds the same +// key/value pair). +// +// - Secret: cannot be scoped either way. Not by label — gateway certificate +// Secrets are created by the external cert-generator job, unlabeled. No +// longer by field either: this cache previously excluded SA-token Secrets +// with a "type" field selector, but the Standby now needs their shells in +// order to hold a worker credential valid on itself (CredentialMirrorSet, +// ADR #293 Decision 6), and no single selector admits both those and the +// unlabeled certificate Secrets. So Secrets are cached cluster-wide and +// the project-namespace boundary stays client-side, in the row's +// RequireMirroredNamespace gate — a Secret in kube-system is cached but +// never written to the Standby. +// +// What that widening does *not* do is widen access: the Standby's remote +// identity already holds cluster-wide Secret read (config/ha, which says +// so in as many words), because RBAC cannot scope Secrets by type or by +// namespace label. The field selector narrowed what this process cached, +// not what it was allowed to fetch. It does mean Active-side token bytes +// would now pass through this process, so they are stripped on the way +// into the cache by sanitizeCachedSecret below, before anything can read +// them — the mirror's own Sanitize is the correctness layer, this is +// defence in depth. +func mirrorCacheByObject() map[client.Object]cache.ByObject { + controllerManaged := namespaceMirrorSelector() + return map[client.Object]cache.ByObject{ + &corev1.Namespace{}: {Label: controllerManaged}, + &corev1.ServiceAccount{}: {Label: controllerManaged}, + &rbacv1.Role{}: {Label: controllerManaged}, + &rbacv1.RoleBinding{}: {Label: controllerManaged}, + &corev1.Secret{}: {Transform: sanitizeCachedSecret}, + } +} + +// sanitizeCachedSecret drops the token bytes of every service-account-token +// Secret on their way into the remote cache, so an Active-minted token is +// never held in this process at all. Applied to both typed and unstructured +// informers (controller-runtime resolves cache.ByObject by GVK), which is what +// makes it a real boundary rather than a courtesy: the syncer reads through +// the unstructured path. +// +// Only .data is dropped here. The service-account.uid annotation has to +// survive into the cache — prune diffs against this view, and stripping +// identity from the cached copy would make the diff lie — so it is dropped in +// the payload instead, by sanitizeSecret. Anything that is not an SA-token +// Secret, and anything that is not a Secret at all, passes through untouched. +func sanitizeCachedSecret(obj any) (any, error) { + switch secret := obj.(type) { + case *corev1.Secret: + if secret.Type != corev1.SecretTypeServiceAccountToken || secret.Data == nil { + return obj, nil + } + stripped := secret.DeepCopy() + stripped.Data = nil + return stripped, nil + case *unstructured.Unstructured: + if !isServiceAccountTokenSecret(secret) { + return obj, nil + } + if _, found, _ := unstructured.NestedFieldNoCopy(secret.Object, "data"); !found { + return obj, nil + } + stripped := secret.DeepCopy() + delete(stripped.Object, "data") + return stripped, nil + default: + return obj, nil + } +} + +// opDelete extends mirror.go's opCreate/opUpdate for use in this file's +// metrics/logging; mirrorDelete itself has no ambiguity about which +// operation it performed, so it doesn't need to return one. +const opDelete mirrorOp = "delete" + +// remoteGetFunc reads one object from the Active hub by key. The real +// implementation (getFromRemoteCache) reads controller-runtime's cache, which +// is itself a local-indexer read, not network I/O — so retrying via this +// function is cheap even under the workqueue's backoff. Overridable in tests +// so the retry engine is exercised without a real *rest.Config. +type remoteGetFunc func(ctx context.Context, key syncKey) (*unstructured.Unstructured, error) + +// RemoteSyncerOptions configures a RemoteSyncer. Zero-valued fields fall back +// to the Default* constants, matching pkg/ha's existing Options pattern +// (see ClusterLeaderElector's Options in leader_elector.go). +type RemoteSyncerOptions struct { + // Resources is the mirrored-resource table. Defaults to CRDMirrorSet. + Resources []MirroredResource + // Workers is the number of goroutines draining the mirror workqueue. + Workers int + // SetupRetryPeriod is how long Start waits between retries when wiring up + // the remote informers fails. + SetupRetryPeriod time.Duration + // PruneInterval is how often the prune backstop diffs Standby mirrors + // against the Active hub and removes orphans (see prune.go). + PruneInterval time.Duration + // EventRecorder, if set, gets an HAMirrorSyncFailed event on the first + // failure of each mirror-failure episode, attached to the object that + // failed to sync. Nil disables event emission (metrics and retries are + // unaffected). + EventRecorder events.EventRecorder + Log *zap.SugaredLogger +} + +// RemoteSyncer mirrors a fixed set of resources from the Active hub onto the +// Standby's own cluster. It runs only in standby mode: informer event +// handlers enqueue a syncKey (no mirror logic in the callback itself), and a +// small worker pool dequeues, re-reads the object from the Active cache, and +// mirrors it — retrying with backoff via a rate-limited workqueue on any +// failure, the same primitive controller-runtime's own Controller uses +// internally. See ADR #293 and issue #295. +type RemoteSyncer struct { + mode HAMode + localClient client.Client + remoteCache cache.Cache + remoteGet remoteGetFunc + + resources []MirroredResource + byGVK map[schema.GroupVersionKind]MirroredResource + + workers int + setupRetryPeriod time.Duration + pruneInterval time.Duration + // remoteList and waitForCacheSync back the prune loop (prune.go); like + // remoteGet, they are fields so tests can drive pruning without a real + // *rest.Config. + remoteList remoteListFunc + waitForCacheSync func(ctx context.Context) bool + queue workqueue.TypedRateLimitingInterface[syncKey] + // register performs one attempt at wiring informer event handlers for + // every mirrored resource. Kept as a field (rather than calling + // registerInformersOnce directly) so tests can exercise the retry loop + // in Start without a real *rest.Config, the same reason remoteGet exists. + register func(ctx context.Context) error + // handlerRegistered tracks which GVKs already have their event handler + // wired up, so a retried registerInformersOnce (after a later resource in + // the list failed) skips the ones that already succeeded instead of + // calling AddEventHandlerWithResyncPeriod on them again. Informer. + // AddEventHandlerWithResyncPeriod adds an independent handler on every + // call — it is not idempotent like GetInformer — so without this guard a + // retry would double (or triple, ...) that resource's event and resync + // load for the rest of the process's lifetime. Only ever touched from + // Start's single call path, so it needs no locking. + handlerRegistered map[schema.GroupVersionKind]bool + + // enqueuedAt tracks first-enqueue time per key, so update/delete lag + // reflects total time since the triggering change even after a + // coalescing queue collapses repeated events and retries into one item. + mu sync.Mutex + enqueuedAt map[syncKey]time.Time + + eventRecorder events.EventRecorder + log *zap.SugaredLogger +} + +// NewRemoteSyncer builds a RemoteSyncer. remoteCfg and scheme are only used +// (and required) in standby mode, mirroring how NewClusterLeaderElector +// accepts a possibly-nil remote client for non-standby modes. +func NewRemoteSyncer(localClient client.Client, remoteCfg *rest.Config, scheme *runtime.Scheme, mode HAMode, opts RemoteSyncerOptions) (*RemoteSyncer, error) { + if len(opts.Resources) == 0 { + opts.Resources = CRDMirrorSet + } + if opts.Workers == 0 { + opts.Workers = DefaultSyncWorkers + } + if opts.SetupRetryPeriod == 0 { + opts.SetupRetryPeriod = DefaultInformerSetupRetryPeriod + } + if opts.PruneInterval == 0 { + opts.PruneInterval = DefaultPruneInterval + } + if opts.Log == nil { + opts.Log = util.NewLogger().With("name", "ha-remote-syncer") + } + + byGVK := make(map[schema.GroupVersionKind]MirroredResource, len(opts.Resources)) + for _, res := range opts.Resources { + byGVK[res.GVK] = res + } + + s := &RemoteSyncer{ + mode: mode, + localClient: localClient, + resources: opts.Resources, + byGVK: byGVK, + workers: opts.Workers, + setupRetryPeriod: opts.SetupRetryPeriod, + pruneInterval: opts.PruneInterval, + queue: workqueue.NewTypedRateLimitingQueue[syncKey](workqueue.DefaultTypedControllerRateLimiter[syncKey]()), + handlerRegistered: make(map[schema.GroupVersionKind]bool, len(opts.Resources)), + enqueuedAt: make(map[syncKey]time.Time), + eventRecorder: opts.EventRecorder, + log: opts.Log, + } + s.register = s.registerInformersOnce + + if mode == ModeStandby { + if remoteCfg == nil { + return nil, fmt.Errorf("standby mode requires a remote config for the active hub") + } + remoteCache, err := cache.New(remoteCfg, cache.Options{ + Scheme: scheme, + ByObject: mirrorCacheByObject(), + }) + if err != nil { + return nil, fmt.Errorf("building remote cache: %w", err) + } + s.remoteCache = remoteCache + s.remoteGet = s.getFromRemoteCache + s.remoteList = s.listFromRemoteCache + s.waitForCacheSync = remoteCache.WaitForCacheSync + } + + return s, nil +} + +// Start runs RemoteSyncer until ctx is cancelled. It is a no-op in any mode +// other than standby. It returns nil (not ctx.Err()) on graceful shutdown, +// the same contract StartLeaseRenewal/WatchRemoteLease use. +func (s *RemoteSyncer) Start(ctx context.Context) error { + if s.mode != ModeStandby { + s.log.Infow("remote syncer not started; not in standby mode", "mode", s.mode) + return nil + } + + if !s.registerInformers(ctx) { + return nil // ctx cancelled while retrying informer setup + } + + var wg sync.WaitGroup + for i := 0; i < s.workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + s.runWorker(ctx) + }() + } + // The prune loop only watches its context, unlike the workers (which exit + // on queue shutdown) — cancel it explicitly so wg.Wait can't hang if the + // cache ever stops with an error before ctx itself is cancelled. + pruneCtx, cancelPrune := context.WithCancel(ctx) + defer cancelPrune() + wg.Add(1) + go func() { + defer wg.Done() + s.runPrune(pruneCtx) + }() + + s.log.Infow("remote syncer started", "resources", len(s.resources), "workers", s.workers, "pruneInterval", s.pruneInterval) + err := s.remoteCache.Start(ctx) // blocks until ctx.Done(); returns nil on graceful shutdown + cancelPrune() + s.queue.ShutDown() + wg.Wait() + return err +} + +// registerInformers wires up an event handler for every mirrored resource, +// retrying with backoff via s.register on failure — e.g. the Active hub +// briefly unreachable at Standby startup, or its config/ha RBAC not yet +// applied — instead of giving up after one attempt. Without this, a single +// transient failure here would return an error from Start and permanently +// disable mirroring for the process's lifetime (main.go only logs that +// error; nothing restarts the goroutine), unlike StartLeaseRenewal and +// WatchRemoteLease, which retry every tick regardless of error. Returns +// false only if ctx is cancelled before setup succeeds. +func (s *RemoteSyncer) registerInformers(ctx context.Context) bool { + for { + err := s.register(ctx) + if err == nil { + return true + } + s.log.Warnw("remote syncer: informer setup failed; retrying", + "error", err, "retryAfter", s.setupRetryPeriod) + select { + case <-ctx.Done(): + return false + case <-time.After(s.setupRetryPeriod): + } + } +} + +// registerInformersOnce is register's real implementation: a single attempt +// at wiring an event handler for every mirrored resource onto the remote +// cache. Resources whose handler already got registered by an earlier, +// partially-failed attempt are skipped — see handlerRegistered's doc comment +// for why that matters. +func (s *RemoteSyncer) registerInformersOnce(ctx context.Context) error { + for _, res := range s.resources { + if s.handlerRegistered[res.GVK] { + continue + } + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(res.GVK) + inf, err := s.remoteCache.GetInformer(ctx, u) + if err != nil { + return fmt.Errorf("remote syncer: getting informer for %s: %w", res.GVK, err) + } + if _, err := inf.AddEventHandlerWithResyncPeriod(s.handlersFor(res.GVK), DefaultInformerResyncPeriod); err != nil { + return fmt.Errorf("remote syncer: adding handler for %s: %w", res.GVK, err) + } + s.handlerRegistered[res.GVK] = true + } + return nil +} + +// handlersFor returns the informer callbacks for one GVK. They only enqueue a +// syncKey — no mirror logic runs on the informer's own goroutine. A burst of +// Update events for the same object coalesces into one queued item; the +// worker determines the real action at dequeue time by re-reading the Active +// cache (found -> mirror, NotFound -> delete), the same way a Reconcile call +// would. +func (s *RemoteSyncer) handlersFor(objGVK schema.GroupVersionKind) toolscache.ResourceEventHandlerFuncs { + enqueue := func(obj interface{}) { + if tomb, ok := obj.(toolscache.DeletedFinalStateUnknown); ok { + obj = tomb.Obj + } + u, ok := obj.(*unstructured.Unstructured) + if !ok { + s.log.Warnw("remote syncer: unexpected informer object type", "type", fmt.Sprintf("%T", obj)) + return + } + s.enqueue(syncKey{GVK: objGVK, Namespace: u.GetNamespace(), Name: u.GetName()}) + } + return toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { enqueue(obj) }, + UpdateFunc: func(_, newObj interface{}) { enqueue(newObj) }, + DeleteFunc: func(obj interface{}) { enqueue(obj) }, + } +} + +// enqueue hands one key to the worker pool, stamping its first-enqueue time +// for the lag metric. Both the informer handlers and the prune loop go +// through here, so every mirror write shares one queue and one retry policy. +func (s *RemoteSyncer) enqueue(key syncKey) { + s.markEnqueued(key) + s.queue.Add(key) + haSyncQueueDepth.Set(float64(s.queue.Len())) +} + +func (s *RemoteSyncer) runWorker(ctx context.Context) { + for { + key, shutdown := s.queue.Get() + if shutdown { + return + } + s.processOnce(ctx, key) + } +} + +// processOnce dequeues exactly one key and mirrors it. Any error — including +// a namespaced object created before its Namespace has synced, the concrete +// failure mode that motivated this design — is retried with backoff via +// queue.AddRateLimited rather than dropped, satisfying issue #295's own +// acceptance criterion that the syncer retries without crashing. +func (s *RemoteSyncer) processOnce(ctx context.Context, key syncKey) { + // Registered before Done so that it runs after it: defers are LIFO, and + // Done is what re-queues a key that was marked dirty while in flight, so + // sampling afterwards is what makes the depth include that requeue. Sampled + // on the way out as well as on every enqueue, so the gauge falls as a backlog + // drains rather than only ever rising. + defer func() { haSyncQueueDepth.Set(float64(s.queue.Len())) }() + defer s.queue.Done(key) + + op, lagSeconds, err := s.reconcileKey(ctx, key) + if err != nil { + label := string(op) + if label == "" { + label = "sync" + } + haSyncErrorsTotal.WithLabelValues(key.GVK.Kind, label).Inc() + s.log.Warnw("mirror sync failed; will retry", "kind", key.GVK.Kind, + "namespace", key.Namespace, "name", key.Name, + "attempt", s.queue.NumRequeues(key), "error", err) + // One HAMirrorSyncFailed event per failure episode (NumRequeues is + // still 0 here on the first failure; Forget resets it on success). + // The retries that follow are milliseconds apart under early + // backoff, and although the recorder aggregates repeats into one + // Event's Count, every call is still an API-server write — + // per-attempt emission would turn each outage into a write storm + // while ha_sync_errors_total already counts every attempt. + // The recorder is called directly rather than through + // util.RecordEvent: that helper logs via util.CtxLogger, which + // panics on any context that didn't pass through a reconciler's + // PrepareKubeSliceControllersRequestContext — and Start's context + // (main.go's signal-handler context) never does. + if s.eventRecorder != nil && s.queue.NumRequeues(key) == 0 { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(key.GVK) + obj.SetNamespace(key.Namespace) + obj.SetName(key.Name) + if recErr := s.eventRecorder.RecordEvent(ctx, &events.Event{ + Object: obj, + ReportingInstance: util.InstanceController, + Name: ossEvents.EventHAMirrorSyncFailed, + }); recErr != nil { + s.log.Warnw("failed to record mirror-sync-failed event", + "kind", key.GVK.Kind, "namespace", key.Namespace, + "name", key.Name, "error", recErr) + } + } + s.queue.AddRateLimited(key) + return + } + if op != "" { + haSyncLagSeconds.WithLabelValues(key.GVK.Kind, string(op)).Observe(lagSeconds) + } + s.queue.Forget(key) + s.clearEnqueued(key) +} + +// reconcileKey reads the current state of key from the Active hub and +// mirrors it: NotFound -> delete, found -> create-or-update. It returns the +// operation performed ("" if the conflict guard or a Skip predicate +// suppressed it) and the lag to record for that operation. +func (s *RemoteSyncer) reconcileKey(ctx context.Context, key syncKey) (mirrorOp, float64, error) { + res, ok := s.byGVK[key.GVK] + if !ok { + return "", 0, nil + } + + src, err := s.remoteGet(ctx, key) + switch { + case apierrors.IsNotFound(err): + if err := mirrorDelete(ctx, s.localClient, key); err != nil { + return opDelete, 0, err + } + return opDelete, time.Since(s.enqueuedTime(key)).Seconds(), nil + case err != nil: + return "", 0, fmt.Errorf("reading from active cache: %w", err) + } + + if res.Skip != nil && res.Skip(src) { + return "", 0, nil + } + if res.RequireMirroredNamespace { + mirrored, err := s.namespaceIsMirrored(ctx, src.GetNamespace()) + if err != nil { + return "", 0, fmt.Errorf("checking namespace of %s %s/%s: %w", key.GVK.Kind, key.Namespace, key.Name, err) + } + if !mirrored { + return "", 0, nil + } + } + + op, err := mirrorCreateOrUpdate(ctx, s.localClient, key, res, src) + if err != nil { + return op, 0, err + } + if op == "" { + return "", 0, nil // conflict guard skipped it + } + if op == opCreate { + return op, time.Since(src.GetCreationTimestamp().Time).Seconds(), nil + } + return op, time.Since(s.enqueuedTime(key)).Seconds(), nil +} + +// namespaceIsMirrored reports whether ns is one of the namespaces the syncer +// itself mirrors, by reading the remote cache's Namespace view — which is +// label-scoped to controller-managed project namespaces (see +// namespaceMirrorSelector), so any namespace outside that boundary reads as +// NotFound here no matter what it is named. This is the namespace gate +// behind MirroredResource.RequireMirroredNamespace. +// +// A skip verdict is terminal for this queue item (no retry), so an object +// racing its own namespace's informer delivery on cold start can be skipped +// once — the prune loop's reverse diff re-enqueues it within one +// --ha-sync-interval (see pruneOnce), rather than waiting for the informer's +// much longer resync period. +func (s *RemoteSyncer) namespaceIsMirrored(ctx context.Context, ns string) (bool, error) { + if ns == "" { + return true, nil // cluster-scoped objects have no namespace to gate on + } + nsKey := syncKey{GVK: schema.GroupVersionKind{Version: "v1", Kind: "Namespace"}, Name: ns} + if _, err := s.remoteGet(ctx, nsKey); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +// getFromRemoteCache is remoteGetFunc's real implementation: a read from +// controller-runtime's cache, which serves Get from the informer's local +// indexer rather than the network, so it stays cheap even when a retry +// re-reads the same key minutes later under backoff. +func (s *RemoteSyncer) getFromRemoteCache(ctx context.Context, key syncKey) (*unstructured.Unstructured, error) { + u := &unstructured.Unstructured{} + u.SetGroupVersionKind(key.GVK) + if err := s.remoteCache.Get(ctx, types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, u); err != nil { + return nil, err + } + return u, nil +} + +func (s *RemoteSyncer) markEnqueued(key syncKey) { + s.mu.Lock() + defer s.mu.Unlock() + if _, exists := s.enqueuedAt[key]; !exists { + s.enqueuedAt[key] = time.Now() + } +} + +func (s *RemoteSyncer) clearEnqueued(key syncKey) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.enqueuedAt, key) +} + +func (s *RemoteSyncer) enqueuedTime(key syncKey) time.Time { + s.mu.Lock() + defer s.mu.Unlock() + if t, ok := s.enqueuedAt[key]; ok { + return t + } + return time.Now() +} diff --git a/pkg/ha/remote_syncer_test.go b/pkg/ha/remote_syncer_test.go new file mode 100644 index 000000000..5105da27d --- /dev/null +++ b/pkg/ha/remote_syncer_test.go @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ha + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + toolscache "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubeslice/kubeslice-controller/util" +) + +// stubRemote is a remoteGetFunc backend the retry-engine tests drive +// directly, so they exercise RemoteSyncer's workqueue/retry logic without a +// real *rest.Config or live cluster. +type stubRemote struct { + mu sync.Mutex + objects map[syncKey]*unstructured.Unstructured + errs map[syncKey]error + calls map[syncKey]int +} + +func newStubRemote() *stubRemote { + return &stubRemote{ + objects: map[syncKey]*unstructured.Unstructured{}, + errs: map[syncKey]error{}, + calls: map[syncKey]int{}, + } +} + +func (s *stubRemote) get(_ context.Context, key syncKey) (*unstructured.Unstructured, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.calls[key]++ + if err, ok := s.errs[key]; ok { + return nil, err + } + if obj, ok := s.objects[key]; ok { + return obj, nil + } + return nil, apierrors.NewNotFound(schema.GroupResource{Group: key.GVK.Group, Resource: key.GVK.Kind}, key.Name) +} + +func (s *stubRemote) callCount(key syncKey) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls[key] +} + +// buildSyncer constructs a RemoteSyncer by struct literal (in-package test), +// backed by remote and a fast-backoff queue, bypassing NewRemoteSyncer's +// cache.New so no *rest.Config is needed. +func buildSyncer(t *testing.T, remote *stubRemote) *RemoteSyncer { + t.Helper() + return &RemoteSyncer{ + mode: ModeStandby, + localClient: mirrorFakeClient(t), + remoteGet: remote.get, + resources: []MirroredResource{{GVK: testGVK}}, + byGVK: map[schema.GroupVersionKind]MirroredResource{testGVK: {GVK: testGVK}}, + workers: 1, + queue: workqueue.NewTypedRateLimitingQueue[syncKey]( + workqueue.NewTypedItemExponentialFailureRateLimiter[syncKey](time.Millisecond, time.Second), + ), + handlerRegistered: map[schema.GroupVersionKind]bool{}, + enqueuedAt: map[syncKey]time.Time{}, + log: testLog(), + } +} + +// stubInformer is a minimal cache.Informer fake that only counts +// AddEventHandlerWithResyncPeriod calls; every other method panics via the +// embedded nil interface if exercised (registerInformersOnce never touches +// them). +type stubInformer struct { + cache.Informer + addCalls int +} + +func (i *stubInformer) AddEventHandlerWithResyncPeriod(_ toolscache.ResourceEventHandler, _ time.Duration) (toolscache.ResourceEventHandlerRegistration, error) { + i.addCalls++ + return nil, nil +} + +// stubCache is a minimal cache.Cache fake that only implements GetInformer, +// letting tests drive registerInformersOnce's retry/dedup behaviour without a +// real *rest.Config. +type stubCache struct { + cache.Cache + informer *stubInformer + failGVKs map[schema.GroupVersionKind]int // remaining failures before success, per GVK + getInformerCalls map[schema.GroupVersionKind]int +} + +func (c *stubCache) GetInformer(_ context.Context, obj client.Object, _ ...cache.InformerGetOption) (cache.Informer, error) { + gvk := obj.GetObjectKind().GroupVersionKind() + c.getInformerCalls[gvk]++ + if c.failGVKs[gvk] > 0 { + c.failGVKs[gvk]-- + return nil, fmt.Errorf("simulated GetInformer failure for %s", gvk) + } + return c.informer, nil +} + +func TestRemoteSyncer_ReconcileKey_FoundMirrorsCreate(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + remote := newStubRemote() + remote.objects[key] = newTestUnstructured(testGVK, key.Namespace, key.Name) + + s := buildSyncer(t, remote) + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, opCreate, op) + + got := getUnstructured(t, s.localClient, key) + assert.Equal(t, LabelValueActive, got.GetLabels()[LabelSyncedFromActive]) +} + +func TestRemoteSyncer_ReconcileKey_NotFoundMirrorsDelete(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + remote := newStubRemote() // nothing registered -> NotFound + + s := buildSyncer(t, remote) + existing := newTestUnstructured(testGVK, key.Namespace, key.Name) + existing.SetLabels(map[string]string{LabelSyncedFromActive: LabelValueActive}) + require.NoError(t, s.localClient.Create(ctx, existing)) + + op, _, err := s.reconcileKey(ctx, key) + require.NoError(t, err) + assert.Equal(t, opDelete, op) + + got := &unstructured.Unstructured{} + got.SetGroupVersionKind(key.GVK) + err = s.localClient.Get(ctx, types.NamespacedName{Namespace: key.Namespace, Name: key.Name}, got) + assert.True(t, apierrors.IsNotFound(err), "the Standby mirror should be gone once Active reports NotFound") +} + +func TestRemoteSyncer_ProcessOnce_RetriesOnErrorAndRedelivers(t *testing.T) { + ctx := context.Background() + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + remote := newStubRemote() + remote.errs[key] = fmt.Errorf("simulated transient read failure") + + s := buildSyncer(t, remote) + s.queue.Add(key) + + got, shutdown := s.queue.Get() + require.False(t, shutdown) + require.Equal(t, key, got) + s.processOnce(ctx, got) + + // AddRateLimited schedules redelivery asynchronously; poll briefly for it + // rather than sleeping a fixed guess. + deadline := time.Now().Add(2 * time.Second) + redelivered := false + for time.Now().Before(deadline) { + if s.queue.Len() > 0 { + redelivered = true + break + } + time.Sleep(5 * time.Millisecond) + } + assert.True(t, redelivered, "a failed sync must be retried, not dropped (issue #295's own acceptance criterion)") + assert.GreaterOrEqual(t, remote.callCount(key), 1) +} + +func TestRemoteSyncer_HandlersFor_UnwrapsDeletedFinalStateUnknown(t *testing.T) { + s := buildSyncer(t, newStubRemote()) + handlers := s.handlersFor(testGVK) + + u := newTestUnstructured(testGVK, "proj-a", "sc-1") + handlers.DeleteFunc(toolscache.DeletedFinalStateUnknown{Key: "proj-a/sc-1", Obj: u}) + + key := syncKey{GVK: testGVK, Namespace: "proj-a", Name: "sc-1"} + require.Equal(t, 1, s.queue.Len()) + got, _ := s.queue.Get() + assert.Equal(t, key, got) +} + +func TestRemoteSyncer_HandlersFor_IgnoresUnexpectedType(t *testing.T) { + s := buildSyncer(t, newStubRemote()) + handlers := s.handlersFor(testGVK) + handlers.AddFunc("not-an-unstructured") + assert.Equal(t, 0, s.queue.Len()) +} + +func TestRemoteSyncer_Start_NoopWhenNotStandby(t *testing.T) { + c := mirrorFakeClient(t) + s, err := NewRemoteSyncer(c, nil, nil, ModeStandalone, RemoteSyncerOptions{Log: testLog()}) + require.NoError(t, err) + assert.NoError(t, s.Start(context.Background())) +} + +func TestNewRemoteSyncer_StandbyRequiresRemoteConfig(t *testing.T) { + c := mirrorFakeClient(t) + _, err := NewRemoteSyncer(c, nil, testScheme(t), ModeStandby, RemoteSyncerOptions{Log: testLog()}) + assert.Error(t, err) +} + +func TestNamespaceMirrorSelector_MatchesOnlyProjectNamespaces(t *testing.T) { + sel := namespaceMirrorSelector() + assert.True(t, sel.Matches(labels.Set(util.LabelsKubeSliceController)), + "selector must match the labels NamespaceService.ReconcileProjectNamespace actually stamps on project namespaces") + assert.False(t, sel.Matches(labels.Set{"kubernetes.io/metadata.name": "kube-system"}), + "selector must not match an unrelated system namespace that happens to exist on the Active hub") +} + +func TestRegisterInformers_RetriesUntilSuccess(t *testing.T) { + s := buildSyncer(t, newStubRemote()) + s.setupRetryPeriod = time.Millisecond + + var calls int32 + s.register = func(_ context.Context) error { + if atomic.AddInt32(&calls, 1) < 3 { + return fmt.Errorf("simulated transient informer setup failure") + } + return nil + } + + ok := s.registerInformers(context.Background()) + assert.True(t, ok, "registerInformers must keep retrying instead of giving up on the first failure") + assert.GreaterOrEqual(t, atomic.LoadInt32(&calls), int32(3)) +} + +func TestRegisterInformersOnce_SkipsAlreadyRegisteredHandlersOnRetry(t *testing.T) { + resA := MirroredResource{GVK: schema.GroupVersionKind{Group: groupController, Version: "v1alpha1", Kind: "Cluster"}} + resB := MirroredResource{GVK: schema.GroupVersionKind{Group: groupController, Version: "v1alpha1", Kind: "Project"}} + + informer := &stubInformer{} + sc := &stubCache{ + informer: informer, + failGVKs: map[schema.GroupVersionKind]int{resB.GVK: 1}, // fails once, then succeeds + getInformerCalls: map[schema.GroupVersionKind]int{}, + } + + s := buildSyncer(t, newStubRemote()) + s.resources = []MirroredResource{resA, resB} + s.remoteCache = sc + + // First attempt: resA succeeds and its handler gets registered; resB + // fails at GetInformer, so registerInformersOnce returns an error before + // reaching the end of the resource list. + err := s.registerInformersOnce(context.Background()) + require.Error(t, err) + assert.Equal(t, 1, informer.addCalls, "resA's handler should be registered exactly once after the first (partial) attempt") + + // Second attempt (what registerInformers' retry loop would do): resA + // must NOT be re-registered — AddEventHandlerWithResyncPeriod is not + // idempotent, so a naive from-scratch retry would double resA's event + // and resync load. resB now succeeds and gets registered for the first time. + err = s.registerInformersOnce(context.Background()) + require.NoError(t, err) + assert.Equal(t, 2, informer.addCalls, "retry must add resB's handler but must not double-register resA's") +} + +func TestRegisterInformers_StopsRetryingOnContextCancel(t *testing.T) { + s := buildSyncer(t, newStubRemote()) + s.setupRetryPeriod = 50 * time.Millisecond + s.register = func(_ context.Context) error { + return fmt.Errorf("always fails") + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan bool, 1) + go func() { done <- s.registerInformers(ctx) }() + + time.Sleep(10 * time.Millisecond) + cancel() + + select { + case ok := <-done: + assert.False(t, ok, "registerInformers must report failure when ctx is cancelled mid-retry") + case <-time.After(2 * time.Second): + t.Fatal("registerInformers did not return after context cancellation") + } +} diff --git a/service/access_control_service.go b/service/access_control_service.go index 7c2bc4582..64b73dc58 100644 --- a/service/access_control_service.go +++ b/service/access_control_service.go @@ -427,11 +427,44 @@ func (a *AccessControlService) createOrUpdateServiceAccountsAndRoleBindings(ctx "object_kind": metricKindServiceAccount, }, ) + } + + // The token Secret's existence is checked independently of the + // ServiceAccount's, and must stay that way. Creating it only inside the + // branch above assumes the two are always absent together, which is true + // when this routine created both — and false as soon as a ServiceAccount + // arrives by any other means. A cross-cluster HA Standby is the case that + // exposes it: the state mirror copies ServiceAccounts, and copies their + // token Secrets only as empty shells — a token signed by one cluster is + // invalid on another, so the value never crosses and each cluster's own + // token controller fills its own copy in. A promoted hub that finds the + // account present would skip the branch, never mint a token, and then + // fail every reconcile of every registered cluster on the missing Secret. + // The requeue guard in ClusterService.ReconcileCluster does not catch it + // either, because the ServiceAccount is built with its Secrets reference + // already populated and therefore claims a Secret that does not exist. + // + // This is behaviour-neutral on a hub that has only ever created its own + // accounts, where the Secret is always present whenever the account is, + // and on a promoted Standby that mirrored the shell — the check below + // simply finds it. It remains the recovery path for the cases the mirror + // cannot cover: a hub promoted before shell mirroring existed, and any + // cluster whose token Secret was deleted by hand. + secretNamespacedName := client.ObjectKey{ + Namespace: namespace, + Name: serviceAccountNamespacedName.Name, + } + foundSecret, err := util.GetResourceIfExist(ctx, secretNamespacedName, &corev1.Secret{}) + if err != nil { + logger.With(zap.Error(err)).Errorf("Couldnt fetch service account secret") + return ctrl.Result{}, err + } + if !foundSecret { // create secret for the service account secret := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: expectedServiceAccount.Name, - Annotations: map[string]string{"kubernetes.io/service-account.name": expectedServiceAccount.Name}, + Name: serviceAccountNamespacedName.Name, + Annotations: map[string]string{"kubernetes.io/service-account.name": serviceAccountNamespacedName.Name}, Namespace: namespace, }, Type: "kubernetes.io/service-account-token", diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/lint.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/lint.go new file mode 100644 index 000000000..8d2f05500 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/lint.go @@ -0,0 +1,46 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "fmt" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil/promlint" +) + +// CollectAndLint registers the provided Collector with a newly created pedantic +// Registry. It then calls GatherAndLint with that Registry and with the +// provided metricNames. +func CollectAndLint(c prometheus.Collector, metricNames ...string) ([]promlint.Problem, error) { + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + return nil, fmt.Errorf("registering collector failed: %w", err) + } + return GatherAndLint(reg, metricNames...) +} + +// GatherAndLint gathers all metrics from the provided Gatherer and checks them +// with the linter in the promlint package. If any metricNames are provided, +// only metrics with those names are checked. +func GatherAndLint(g prometheus.Gatherer, metricNames ...string) ([]promlint.Problem, error) { + got, err := g.Gather() + if err != nil { + return nil, fmt.Errorf("gathering metrics failed: %w", err) + } + if metricNames != nil { + got = filterMetrics(got, metricNames) + } + return promlint.NewWithMetricFamilies(got).Lint() +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/problem.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/problem.go new file mode 100644 index 000000000..9ba42826a --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/problem.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promlint + +import dto "github.com/prometheus/client_model/go" + +// A Problem is an issue detected by a linter. +type Problem struct { + // The name of the metric indicated by this Problem. + Metric string + + // A description of the issue for this Problem. + Text string +} + +// newProblem is helper function to create a Problem. +func newProblem(mf *dto.MetricFamily, text string) Problem { + return Problem{ + Metric: mf.GetName(), + Text: text, + } +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/promlint.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/promlint.go new file mode 100644 index 000000000..ea46f38ec --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/promlint.go @@ -0,0 +1,123 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package promlint provides a linter for Prometheus metrics. +package promlint + +import ( + "errors" + "io" + "sort" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +// A Linter is a Prometheus metrics linter. It identifies issues with metric +// names, types, and metadata, and reports them to the caller. +type Linter struct { + // The linter will read metrics in the Prometheus text format from r and + // then lint it, _and_ it will lint the metrics provided directly as + // MetricFamily proto messages in mfs. Note, however, that the current + // constructor functions New and NewWithMetricFamilies only ever set one + // of them. + r io.Reader + mfs []*dto.MetricFamily + + customValidations []Validation +} + +// New creates a new Linter that reads an input stream of Prometheus metrics in +// the Prometheus text exposition format. +func New(r io.Reader) *Linter { + return &Linter{ + r: r, + } +} + +// NewWithMetricFamilies creates a new Linter that reads from a slice of +// MetricFamily protobuf messages. +func NewWithMetricFamilies(mfs []*dto.MetricFamily) *Linter { + return &Linter{ + mfs: mfs, + } +} + +// AddCustomValidations adds custom validations to the linter. +func (l *Linter) AddCustomValidations(vs ...Validation) { + if l.customValidations == nil { + l.customValidations = make([]Validation, 0, len(vs)) + } + l.customValidations = append(l.customValidations, vs...) +} + +// Lint performs a linting pass, returning a slice of Problems indicating any +// issues found in the metrics stream. The slice is sorted by metric name +// and issue description. +func (l *Linter) Lint() ([]Problem, error) { + var problems []Problem + + if l.r != nil { + d := expfmt.NewDecoder(l.r, expfmt.NewFormat(expfmt.TypeTextPlain)) + + mf := &dto.MetricFamily{} + for { + if err := d.Decode(mf); err != nil { + if errors.Is(err, io.EOF) { + break + } + + return nil, err + } + + problems = append(problems, l.lint(mf)...) + } + } + for _, mf := range l.mfs { + problems = append(problems, l.lint(mf)...) + } + + // Ensure deterministic output. + sort.SliceStable(problems, func(i, j int) bool { + if problems[i].Metric == problems[j].Metric { + return problems[i].Text < problems[j].Text + } + return problems[i].Metric < problems[j].Metric + }) + + return problems, nil +} + +// lint is the entry point for linting a single metric. +func (l *Linter) lint(mf *dto.MetricFamily) []Problem { + var problems []Problem + + for _, fn := range defaultValidations { + errs := fn(mf) + for _, err := range errs { + problems = append(problems, newProblem(mf, err.Error())) + } + } + + if l.customValidations != nil { + for _, fn := range l.customValidations { + errs := fn(mf) + for _, err := range errs { + problems = append(problems, newProblem(mf, err.Error())) + } + } + } + + // TODO(mdlayher): lint rules for specific metrics types. + return problems +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validation.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validation.go new file mode 100644 index 000000000..f52ad9eab --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validation.go @@ -0,0 +1,33 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package promlint + +import ( + dto "github.com/prometheus/client_model/go" + + "github.com/prometheus/client_golang/prometheus/testutil/promlint/validations" +) + +type Validation = func(mf *dto.MetricFamily) []error + +var defaultValidations = []Validation{ + validations.LintHelp, + validations.LintMetricUnits, + validations.LintCounter, + validations.LintHistogramSummaryReserved, + validations.LintMetricTypeInName, + validations.LintReservedChars, + validations.LintCamelCase, + validations.LintUnitAbbreviations, +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/counter_validations.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/counter_validations.go new file mode 100644 index 000000000..f2c2c3905 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/counter_validations.go @@ -0,0 +1,40 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "errors" + "strings" + + dto "github.com/prometheus/client_model/go" +) + +// LintCounter detects issues specific to counters, as well as patterns that should +// only be used with counters. +func LintCounter(mf *dto.MetricFamily) []error { + var problems []error + + isCounter := mf.GetType() == dto.MetricType_COUNTER + isUntyped := mf.GetType() == dto.MetricType_UNTYPED + hasTotalSuffix := strings.HasSuffix(mf.GetName(), "_total") + + switch { + case isCounter && !hasTotalSuffix: + problems = append(problems, errors.New(`counter metrics should have "_total" suffix`)) + case !isUntyped && !isCounter && hasTotalSuffix: + problems = append(problems, errors.New(`non-counter metrics should not have "_total" suffix`)) + } + + return problems +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/generic_name_validations.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/generic_name_validations.go new file mode 100644 index 000000000..bc8dbd1e1 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/generic_name_validations.go @@ -0,0 +1,101 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "errors" + "fmt" + "regexp" + "strings" + + dto "github.com/prometheus/client_model/go" +) + +var camelCase = regexp.MustCompile(`[a-z][A-Z]`) + +// LintMetricUnits detects issues with metric unit names. +func LintMetricUnits(mf *dto.MetricFamily) []error { + var problems []error + + unit, base, ok := metricUnits(*mf.Name) + if !ok { + // No known units detected. + return nil + } + + // Unit is already a base unit. + if unit == base { + return nil + } + + problems = append(problems, fmt.Errorf("use base unit %q instead of %q", base, unit)) + + return problems +} + +// LintMetricTypeInName detects when metric types are included in the metric name. +func LintMetricTypeInName(mf *dto.MetricFamily) []error { + var problems []error + n := strings.ToLower(mf.GetName()) + + for i, t := range dto.MetricType_name { + if i == int32(dto.MetricType_UNTYPED) { + continue + } + + typename := strings.ToLower(t) + if strings.Contains(n, "_"+typename+"_") || strings.HasSuffix(n, "_"+typename) { + problems = append(problems, fmt.Errorf(`metric name should not include type '%s'`, typename)) + } + } + return problems +} + +// LintReservedChars detects colons in metric names. +func LintReservedChars(mf *dto.MetricFamily) []error { + var problems []error + if strings.Contains(mf.GetName(), ":") { + problems = append(problems, errors.New("metric names should not contain ':'")) + } + return problems +} + +// LintCamelCase detects metric names and label names written in camelCase. +func LintCamelCase(mf *dto.MetricFamily) []error { + var problems []error + if camelCase.FindString(mf.GetName()) != "" { + problems = append(problems, errors.New("metric names should be written in 'snake_case' not 'camelCase'")) + } + + for _, m := range mf.GetMetric() { + for _, l := range m.GetLabel() { + if camelCase.FindString(l.GetName()) != "" { + problems = append(problems, errors.New("label names should be written in 'snake_case' not 'camelCase'")) + } + } + } + return problems +} + +// LintUnitAbbreviations detects abbreviated units in the metric name. +func LintUnitAbbreviations(mf *dto.MetricFamily) []error { + var problems []error + n := strings.ToLower(mf.GetName()) + for _, s := range unitAbbreviations { + if strings.Contains(n, "_"+s+"_") || strings.HasSuffix(n, "_"+s) { + problems = append(problems, errors.New("metric names should not contain abbreviated units")) + } + } + return problems +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/help_validations.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/help_validations.go new file mode 100644 index 000000000..1df294468 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/help_validations.go @@ -0,0 +1,32 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "errors" + + dto "github.com/prometheus/client_model/go" +) + +// LintHelp detects issues related to the help text for a metric. +func LintHelp(mf *dto.MetricFamily) []error { + var problems []error + + // Expect all metrics to have help text available. + if mf.Help == nil { + problems = append(problems, errors.New("no help text")) + } + + return problems +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/histogram_validations.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/histogram_validations.go new file mode 100644 index 000000000..6564bdf36 --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/histogram_validations.go @@ -0,0 +1,63 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import ( + "errors" + "strings" + + dto "github.com/prometheus/client_model/go" +) + +// LintHistogramSummaryReserved detects when other types of metrics use names or labels +// reserved for use by histograms and/or summaries. +func LintHistogramSummaryReserved(mf *dto.MetricFamily) []error { + // These rules do not apply to untyped metrics. + t := mf.GetType() + if t == dto.MetricType_UNTYPED { + return nil + } + + var problems []error + + isHistogram := t == dto.MetricType_HISTOGRAM + isSummary := t == dto.MetricType_SUMMARY + + n := mf.GetName() + + if !isHistogram && strings.HasSuffix(n, "_bucket") { + problems = append(problems, errors.New(`non-histogram metrics should not have "_bucket" suffix`)) + } + if !isHistogram && !isSummary && strings.HasSuffix(n, "_count") { + problems = append(problems, errors.New(`non-histogram and non-summary metrics should not have "_count" suffix`)) + } + if !isHistogram && !isSummary && strings.HasSuffix(n, "_sum") { + problems = append(problems, errors.New(`non-histogram and non-summary metrics should not have "_sum" suffix`)) + } + + for _, m := range mf.GetMetric() { + for _, l := range m.GetLabel() { + ln := l.GetName() + + if !isHistogram && ln == "le" { + problems = append(problems, errors.New(`non-histogram metrics should not have "le" label`)) + } + if !isSummary && ln == "quantile" { + problems = append(problems, errors.New(`non-summary metrics should not have "quantile" label`)) + } + } + } + + return problems +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/units.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/units.go new file mode 100644 index 000000000..967977d2b --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/promlint/validations/units.go @@ -0,0 +1,118 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validations + +import "strings" + +// Units and their possible prefixes recognized by this library. More can be +// added over time as needed. +var ( + // map a unit to the appropriate base unit. + units = map[string]string{ + // Base units. + "amperes": "amperes", + "bytes": "bytes", + "celsius": "celsius", // Also allow Celsius because it is common in typical Prometheus use cases. + "grams": "grams", + "joules": "joules", + "kelvin": "kelvin", // SI base unit, used in special cases (e.g. color temperature, scientific measurements). + "meters": "meters", // Both American and international spelling permitted. + "metres": "metres", + "seconds": "seconds", + "volts": "volts", + + // Non base units. + // Time. + "minutes": "seconds", + "hours": "seconds", + "days": "seconds", + "weeks": "seconds", + // Temperature. + "kelvins": "kelvin", + "fahrenheit": "celsius", + "rankine": "celsius", + // Length. + "inches": "meters", + "yards": "meters", + "miles": "meters", + // Bytes. + "bits": "bytes", + // Energy. + "calories": "joules", + // Mass. + "pounds": "grams", + "ounces": "grams", + } + + unitPrefixes = []string{ + "pico", + "nano", + "micro", + "milli", + "centi", + "deci", + "deca", + "hecto", + "kilo", + "kibi", + "mega", + "mibi", + "giga", + "gibi", + "tera", + "tebi", + "peta", + "pebi", + } + + // Common abbreviations that we'd like to discourage. + unitAbbreviations = []string{ + "s", + "ms", + "us", + "ns", + "sec", + "b", + "kb", + "mb", + "gb", + "tb", + "pb", + "m", + "h", + "d", + } +) + +// metricUnits attempts to detect known unit types used as part of a metric name, +// e.g. "foo_bytes_total" or "bar_baz_milligrams". +func metricUnits(m string) (unit, base string, ok bool) { + ss := strings.Split(m, "_") + + for _, s := range ss { + if base, found := units[s]; found { + return s, base, true + } + + for _, p := range unitPrefixes { + if strings.HasPrefix(s, p) { + if base, found := units[s[len(p):]]; found { + return s, base, true + } + } + } + } + + return "", "", false +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/testutil/testutil.go b/vendor/github.com/prometheus/client_golang/prometheus/testutil/testutil.go new file mode 100644 index 000000000..9dce15eaf --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/testutil/testutil.go @@ -0,0 +1,358 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package testutil provides helpers to test code using the prometheus package +// of client_golang. +// +// While writing unit tests to verify correct instrumentation of your code, it's +// a common mistake to mostly test the instrumentation library instead of your +// own code. Rather than verifying that a prometheus.Counter's value has changed +// as expected or that it shows up in the exposition after registration, it is +// in general more robust and more faithful to the concept of unit tests to use +// mock implementations of the prometheus.Counter and prometheus.Registerer +// interfaces that simply assert that the Add or Register methods have been +// called with the expected arguments. However, this might be overkill in simple +// scenarios. The ToFloat64 function is provided for simple inspection of a +// single-value metric, but it has to be used with caution. +// +// End-to-end tests to verify all or larger parts of the metrics exposition can +// be implemented with the CollectAndCompare or GatherAndCompare functions. The +// most appropriate use is not so much testing instrumentation of your code, but +// testing custom prometheus.Collector implementations and in particular whole +// exporters, i.e. programs that retrieve telemetry data from a 3rd party source +// and convert it into Prometheus metrics. +// +// In a similar pattern, CollectAndLint and GatherAndLint can be used to detect +// metrics that have issues with their name, type, or metadata without being +// necessarily invalid, e.g. a counter with a name missing the “_total” suffix. +package testutil + +import ( + "bytes" + "fmt" + "io" + "net/http" + "reflect" + + "github.com/davecgh/go-spew/spew" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "google.golang.org/protobuf/proto" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/internal" +) + +// ToFloat64 collects all Metrics from the provided Collector. It expects that +// this results in exactly one Metric being collected, which must be a Gauge, +// Counter, or Untyped. In all other cases, ToFloat64 panics. ToFloat64 returns +// the value of the collected Metric. +// +// The Collector provided is typically a simple instance of Gauge or Counter, or +// – less commonly – a GaugeVec or CounterVec with exactly one element. But any +// Collector fulfilling the prerequisites described above will do. +// +// Use this function with caution. It is computationally very expensive and thus +// not suited at all to read values from Metrics in regular code. This is really +// only for testing purposes, and even for testing, other approaches are often +// more appropriate (see this package's documentation). +// +// A clear anti-pattern would be to use a metric type from the prometheus +// package to track values that are also needed for something else than the +// exposition of Prometheus metrics. For example, you would like to track the +// number of items in a queue because your code should reject queuing further +// items if a certain limit is reached. It is tempting to track the number of +// items in a prometheus.Gauge, as it is then easily available as a metric for +// exposition, too. However, then you would need to call ToFloat64 in your +// regular code, potentially quite often. The recommended way is to track the +// number of items conventionally (in the way you would have done it without +// considering Prometheus metrics) and then expose the number with a +// prometheus.GaugeFunc. +func ToFloat64(c prometheus.Collector) float64 { + var ( + m prometheus.Metric + mCount int + mChan = make(chan prometheus.Metric) + done = make(chan struct{}) + ) + + go func() { + for m = range mChan { + mCount++ + } + close(done) + }() + + c.Collect(mChan) + close(mChan) + <-done + + if mCount != 1 { + panic(fmt.Errorf("collected %d metrics instead of exactly 1", mCount)) + } + + pb := &dto.Metric{} + if err := m.Write(pb); err != nil { + panic(fmt.Errorf("error happened while collecting metrics: %w", err)) + } + if pb.Gauge != nil { + return pb.Gauge.GetValue() + } + if pb.Counter != nil { + return pb.Counter.GetValue() + } + if pb.Untyped != nil { + return pb.Untyped.GetValue() + } + panic(fmt.Errorf("collected a non-gauge/counter/untyped metric: %s", pb)) +} + +// CollectAndCount registers the provided Collector with a newly created +// pedantic Registry. It then calls GatherAndCount with that Registry and with +// the provided metricNames. In the unlikely case that the registration or the +// gathering fails, this function panics. (This is inconsistent with the other +// CollectAnd… functions in this package and has historical reasons. Changing +// the function signature would be a breaking change and will therefore only +// happen with the next major version bump.) +func CollectAndCount(c prometheus.Collector, metricNames ...string) int { + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + panic(fmt.Errorf("registering collector failed: %w", err)) + } + result, err := GatherAndCount(reg, metricNames...) + if err != nil { + panic(err) + } + return result +} + +// GatherAndCount gathers all metrics from the provided Gatherer and counts +// them. It returns the number of metric children in all gathered metric +// families together. If any metricNames are provided, only metrics with those +// names are counted. +func GatherAndCount(g prometheus.Gatherer, metricNames ...string) (int, error) { + got, err := g.Gather() + if err != nil { + return 0, fmt.Errorf("gathering metrics failed: %w", err) + } + if metricNames != nil { + got = filterMetrics(got, metricNames) + } + + result := 0 + for _, mf := range got { + result += len(mf.GetMetric()) + } + return result, nil +} + +// ScrapeAndCompare calls a remote exporter's endpoint which is expected to return some metrics in +// plain text format. Then it compares it with the results that the `expected` would return. +// If the `metricNames` is not empty it would filter the comparison only to the given metric names. +func ScrapeAndCompare(url string, expected io.Reader, metricNames ...string) error { + resp, err := http.Get(url) + if err != nil { + return fmt.Errorf("scraping metrics failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("the scraping target returned a status code other than 200: %d", + resp.StatusCode) + } + + scraped, err := convertReaderToMetricFamily(resp.Body) + if err != nil { + return err + } + + wanted, err := convertReaderToMetricFamily(expected) + if err != nil { + return err + } + + return compareMetricFamilies(scraped, wanted, metricNames...) +} + +// CollectAndCompare registers the provided Collector with a newly created +// pedantic Registry. It then calls GatherAndCompare with that Registry and with +// the provided metricNames. +func CollectAndCompare(c prometheus.Collector, expected io.Reader, metricNames ...string) error { + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(c); err != nil { + return fmt.Errorf("registering collector failed: %w", err) + } + return GatherAndCompare(reg, expected, metricNames...) +} + +// GatherAndCompare gathers all metrics from the provided Gatherer and compares +// it to an expected output read from the provided Reader in the Prometheus text +// exposition format. If any metricNames are provided, only metrics with those +// names are compared. +func GatherAndCompare(g prometheus.Gatherer, expected io.Reader, metricNames ...string) error { + return TransactionalGatherAndCompare(prometheus.ToTransactionalGatherer(g), expected, metricNames...) +} + +// TransactionalGatherAndCompare gathers all metrics from the provided Gatherer and compares +// it to an expected output read from the provided Reader in the Prometheus text +// exposition format. If any metricNames are provided, only metrics with those +// names are compared. +func TransactionalGatherAndCompare(g prometheus.TransactionalGatherer, expected io.Reader, metricNames ...string) error { + got, done, err := g.Gather() + defer done() + if err != nil { + return fmt.Errorf("gathering metrics failed: %w", err) + } + + wanted, err := convertReaderToMetricFamily(expected) + if err != nil { + return err + } + + return compareMetricFamilies(got, wanted, metricNames...) +} + +// convertReaderToMetricFamily would read from a io.Reader object and convert it to a slice of +// dto.MetricFamily. +func convertReaderToMetricFamily(reader io.Reader) ([]*dto.MetricFamily, error) { + var tp expfmt.TextParser + notNormalized, err := tp.TextToMetricFamilies(reader) + if err != nil { + return nil, fmt.Errorf("converting reader to metric families failed: %w", err) + } + + // The text protocol handles empty help fields inconsistently. When + // encoding, any non-nil value, include the empty string, produces a + // "# HELP" line. But when decoding, the help field is only set to a + // non-nil value if the "# HELP" line contains a non-empty value. + // + // Because metrics in a registry always have non-nil help fields, populate + // any nil help fields in the parsed metrics with the empty string so that + // when we compare text encodings, the results are consistent. + for _, metric := range notNormalized { + if metric.Help == nil { + metric.Help = proto.String("") + } + } + + return internal.NormalizeMetricFamilies(notNormalized), nil +} + +// compareMetricFamilies would compare 2 slices of metric families, and optionally filters both of +// them to the `metricNames` provided. +func compareMetricFamilies(got, expected []*dto.MetricFamily, metricNames ...string) error { + if metricNames != nil { + got = filterMetrics(got, metricNames) + expected = filterMetrics(expected, metricNames) + } + + return compare(got, expected) +} + +// compare encodes both provided slices of metric families into the text format, +// compares their string message, and returns an error if they do not match. +// The error contains the encoded text of both the desired and the actual +// result. +func compare(got, want []*dto.MetricFamily) error { + var gotBuf, wantBuf bytes.Buffer + enc := expfmt.NewEncoder(&gotBuf, expfmt.NewFormat(expfmt.TypeTextPlain)) + for _, mf := range got { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("encoding gathered metrics failed: %w", err) + } + } + enc = expfmt.NewEncoder(&wantBuf, expfmt.NewFormat(expfmt.TypeTextPlain)) + for _, mf := range want { + if err := enc.Encode(mf); err != nil { + return fmt.Errorf("encoding expected metrics failed: %w", err) + } + } + if diffErr := diff(wantBuf, gotBuf); diffErr != "" { + return fmt.Errorf(diffErr) + } + return nil +} + +// diff returns a diff of both values as long as both are of the same type and +// are a struct, map, slice, array or string. Otherwise it returns an empty string. +func diff(expected, actual interface{}) string { + if expected == nil || actual == nil { + return "" + } + + et, ek := typeAndKind(expected) + at, _ := typeAndKind(actual) + if et != at { + return "" + } + + if ek != reflect.Struct && ek != reflect.Map && ek != reflect.Slice && ek != reflect.Array && ek != reflect.String { + return "" + } + + var e, a string + c := spew.ConfigState{ + Indent: " ", + DisablePointerAddresses: true, + DisableCapacities: true, + SortKeys: true, + } + if et != reflect.TypeOf("") { + e = c.Sdump(expected) + a = c.Sdump(actual) + } else { + e = reflect.ValueOf(expected).String() + a = reflect.ValueOf(actual).String() + } + + diff, _ := internal.GetUnifiedDiffString(internal.UnifiedDiff{ + A: internal.SplitLines(e), + B: internal.SplitLines(a), + FromFile: "metric output does not match expectation; want", + FromDate: "", + ToFile: "got:", + ToDate: "", + Context: 1, + }) + + if diff == "" { + return "" + } + + return "\n\nDiff:\n" + diff +} + +// typeAndKind returns the type and kind of the given interface{} +func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { + t := reflect.TypeOf(v) + k := t.Kind() + + if k == reflect.Ptr { + t = t.Elem() + k = t.Kind() + } + return t, k +} + +func filterMetrics(metrics []*dto.MetricFamily, names []string) []*dto.MetricFamily { + var filtered []*dto.MetricFamily + for _, m := range metrics { + for _, name := range names { + if m.GetName() == name { + filtered = append(filtered, m) + break + } + } + } + return filtered +} diff --git a/vendor/go.uber.org/zap/zaptest/observer/logged_entry.go b/vendor/go.uber.org/zap/zaptest/observer/logged_entry.go new file mode 100644 index 000000000..a4ea7ec36 --- /dev/null +++ b/vendor/go.uber.org/zap/zaptest/observer/logged_entry.go @@ -0,0 +1,39 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package observer + +import "go.uber.org/zap/zapcore" + +// An LoggedEntry is an encoding-agnostic representation of a log message. +// Field availability is context dependant. +type LoggedEntry struct { + zapcore.Entry + Context []zapcore.Field +} + +// ContextMap returns a map for all fields in Context. +func (e LoggedEntry) ContextMap() map[string]interface{} { + encoder := zapcore.NewMapObjectEncoder() + for _, f := range e.Context { + f.AddTo(encoder) + } + return encoder.Fields +} diff --git a/vendor/go.uber.org/zap/zaptest/observer/observer.go b/vendor/go.uber.org/zap/zaptest/observer/observer.go new file mode 100644 index 000000000..f77f1308b --- /dev/null +++ b/vendor/go.uber.org/zap/zaptest/observer/observer.go @@ -0,0 +1,196 @@ +// Copyright (c) 2016-2022 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package observer provides a zapcore.Core that keeps an in-memory, +// encoding-agnostic representation of log entries. It's useful for +// applications that want to unit test their log output without tying their +// tests to a particular output encoding. +package observer // import "go.uber.org/zap/zaptest/observer" + +import ( + "strings" + "sync" + "time" + + "go.uber.org/zap/internal" + "go.uber.org/zap/zapcore" +) + +// ObservedLogs is a concurrency-safe, ordered collection of observed logs. +type ObservedLogs struct { + mu sync.RWMutex + logs []LoggedEntry +} + +// Len returns the number of items in the collection. +func (o *ObservedLogs) Len() int { + o.mu.RLock() + n := len(o.logs) + o.mu.RUnlock() + return n +} + +// All returns a copy of all the observed logs. +func (o *ObservedLogs) All() []LoggedEntry { + o.mu.RLock() + ret := make([]LoggedEntry, len(o.logs)) + copy(ret, o.logs) + o.mu.RUnlock() + return ret +} + +// TakeAll returns a copy of all the observed logs, and truncates the observed +// slice. +func (o *ObservedLogs) TakeAll() []LoggedEntry { + o.mu.Lock() + ret := o.logs + o.logs = nil + o.mu.Unlock() + return ret +} + +// AllUntimed returns a copy of all the observed logs, but overwrites the +// observed timestamps with time.Time's zero value. This is useful when making +// assertions in tests. +func (o *ObservedLogs) AllUntimed() []LoggedEntry { + ret := o.All() + for i := range ret { + ret[i].Time = time.Time{} + } + return ret +} + +// FilterLevelExact filters entries to those logged at exactly the given level. +func (o *ObservedLogs) FilterLevelExact(level zapcore.Level) *ObservedLogs { + return o.Filter(func(e LoggedEntry) bool { + return e.Level == level + }) +} + +// FilterMessage filters entries to those that have the specified message. +func (o *ObservedLogs) FilterMessage(msg string) *ObservedLogs { + return o.Filter(func(e LoggedEntry) bool { + return e.Message == msg + }) +} + +// FilterMessageSnippet filters entries to those that have a message containing the specified snippet. +func (o *ObservedLogs) FilterMessageSnippet(snippet string) *ObservedLogs { + return o.Filter(func(e LoggedEntry) bool { + return strings.Contains(e.Message, snippet) + }) +} + +// FilterField filters entries to those that have the specified field. +func (o *ObservedLogs) FilterField(field zapcore.Field) *ObservedLogs { + return o.Filter(func(e LoggedEntry) bool { + for _, ctxField := range e.Context { + if ctxField.Equals(field) { + return true + } + } + return false + }) +} + +// FilterFieldKey filters entries to those that have the specified key. +func (o *ObservedLogs) FilterFieldKey(key string) *ObservedLogs { + return o.Filter(func(e LoggedEntry) bool { + for _, ctxField := range e.Context { + if ctxField.Key == key { + return true + } + } + return false + }) +} + +// Filter returns a copy of this ObservedLogs containing only those entries +// for which the provided function returns true. +func (o *ObservedLogs) Filter(keep func(LoggedEntry) bool) *ObservedLogs { + o.mu.RLock() + defer o.mu.RUnlock() + + var filtered []LoggedEntry + for _, entry := range o.logs { + if keep(entry) { + filtered = append(filtered, entry) + } + } + return &ObservedLogs{logs: filtered} +} + +func (o *ObservedLogs) add(log LoggedEntry) { + o.mu.Lock() + o.logs = append(o.logs, log) + o.mu.Unlock() +} + +// New creates a new Core that buffers logs in memory (without any encoding). +// It's particularly useful in tests. +func New(enab zapcore.LevelEnabler) (zapcore.Core, *ObservedLogs) { + ol := &ObservedLogs{} + return &contextObserver{ + LevelEnabler: enab, + logs: ol, + }, ol +} + +type contextObserver struct { + zapcore.LevelEnabler + logs *ObservedLogs + context []zapcore.Field +} + +var ( + _ zapcore.Core = (*contextObserver)(nil) + _ internal.LeveledEnabler = (*contextObserver)(nil) +) + +func (co *contextObserver) Level() zapcore.Level { + return zapcore.LevelOf(co.LevelEnabler) +} + +func (co *contextObserver) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if co.Enabled(ent.Level) { + return ce.AddCore(ent, co) + } + return ce +} + +func (co *contextObserver) With(fields []zapcore.Field) zapcore.Core { + return &contextObserver{ + LevelEnabler: co.LevelEnabler, + logs: co.logs, + context: append(co.context[:len(co.context):len(co.context)], fields...), + } +} + +func (co *contextObserver) Write(ent zapcore.Entry, fields []zapcore.Field) error { + all := make([]zapcore.Field, 0, len(fields)+len(co.context)) + all = append(all, co.context...) + all = append(all, fields...) + co.logs.add(LoggedEntry{ent, all}) + return nil +} + +func (co *contextObserver) Sync() error { + return nil +} diff --git a/vendor/k8s.io/apimachinery/pkg/util/rand/rand.go b/vendor/k8s.io/apimachinery/pkg/util/rand/rand.go new file mode 100644 index 000000000..82a473bb1 --- /dev/null +++ b/vendor/k8s.io/apimachinery/pkg/util/rand/rand.go @@ -0,0 +1,127 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package rand provides utilities related to randomization. +package rand + +import ( + "math/rand" + "sync" + "time" +) + +var rng = struct { + sync.Mutex + rand *rand.Rand +}{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), +} + +// Int returns a non-negative pseudo-random int. +func Int() int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Int() +} + +// Intn generates an integer in range [0,max). +// By design this should panic if input is invalid, <= 0. +func Intn(max int) int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Intn(max) +} + +// IntnRange generates an integer in range [min,max). +// By design this should panic if input is invalid, <= 0. +func IntnRange(min, max int) int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Intn(max-min) + min +} + +// IntnRange generates an int64 integer in range [min,max). +// By design this should panic if input is invalid, <= 0. +func Int63nRange(min, max int64) int64 { + rng.Lock() + defer rng.Unlock() + return rng.rand.Int63n(max-min) + min +} + +// Seed seeds the rng with the provided seed. +func Seed(seed int64) { + rng.Lock() + defer rng.Unlock() + + rng.rand = rand.New(rand.NewSource(seed)) +} + +// Perm returns, as a slice of n ints, a pseudo-random permutation of the integers [0,n) +// from the default Source. +func Perm(n int) []int { + rng.Lock() + defer rng.Unlock() + return rng.rand.Perm(n) +} + +const ( + // We omit vowels from the set of available characters to reduce the chances + // of "bad words" being formed. + alphanums = "bcdfghjklmnpqrstvwxz2456789" + // No. of bits required to index into alphanums string. + alphanumsIdxBits = 5 + // Mask used to extract last alphanumsIdxBits of an int. + alphanumsIdxMask = 1<>= alphanumsIdxBits + remaining-- + } + return string(b) +} + +// SafeEncodeString encodes s using the same characters as rand.String. This reduces the chances of bad words and +// ensures that strings generated from hash functions appear consistent throughout the API. +func SafeEncodeString(s string) string { + r := make([]byte, len(s)) + for i, b := range []rune(s) { + r[i] = alphanums[(int(b) % len(alphanums))] + } + return string(r) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 6a8b31364..bd1d2a12b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -228,6 +228,9 @@ github.com/prometheus/client_golang/prometheus/collectors github.com/prometheus/client_golang/prometheus/internal github.com/prometheus/client_golang/prometheus/promauto github.com/prometheus/client_golang/prometheus/promhttp +github.com/prometheus/client_golang/prometheus/testutil +github.com/prometheus/client_golang/prometheus/testutil/promlint +github.com/prometheus/client_golang/prometheus/testutil/promlint/validations # github.com/prometheus/client_model v0.6.1 ## explicit; go 1.19 github.com/prometheus/client_model/go @@ -328,6 +331,7 @@ go.uber.org/zap/internal/exit go.uber.org/zap/internal/pool go.uber.org/zap/internal/stacktrace go.uber.org/zap/zapcore +go.uber.org/zap/zaptest/observer # golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 ## explicit; go 1.20 golang.org/x/exp/constraints @@ -641,6 +645,7 @@ k8s.io/apimachinery/pkg/util/mergepatch k8s.io/apimachinery/pkg/util/naming k8s.io/apimachinery/pkg/util/net k8s.io/apimachinery/pkg/util/portforward +k8s.io/apimachinery/pkg/util/rand k8s.io/apimachinery/pkg/util/remotecommand k8s.io/apimachinery/pkg/util/runtime k8s.io/apimachinery/pkg/util/sets @@ -1040,6 +1045,8 @@ sigs.k8s.io/controller-runtime/pkg/certwatcher/metrics sigs.k8s.io/controller-runtime/pkg/client sigs.k8s.io/controller-runtime/pkg/client/apiutil sigs.k8s.io/controller-runtime/pkg/client/config +sigs.k8s.io/controller-runtime/pkg/client/fake +sigs.k8s.io/controller-runtime/pkg/client/interceptor sigs.k8s.io/controller-runtime/pkg/cluster sigs.k8s.io/controller-runtime/pkg/config sigs.k8s.io/controller-runtime/pkg/controller @@ -1057,6 +1064,7 @@ sigs.k8s.io/controller-runtime/pkg/internal/flock sigs.k8s.io/controller-runtime/pkg/internal/httpserver sigs.k8s.io/controller-runtime/pkg/internal/log sigs.k8s.io/controller-runtime/pkg/internal/metrics +sigs.k8s.io/controller-runtime/pkg/internal/objectutil sigs.k8s.io/controller-runtime/pkg/internal/recorder sigs.k8s.io/controller-runtime/pkg/internal/source sigs.k8s.io/controller-runtime/pkg/internal/syncs diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go new file mode 100644 index 000000000..793219c72 --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/client.go @@ -0,0 +1,1593 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "runtime/debug" + "strconv" + "strings" + "sync" + "time" + + // Using v4 to match upstream + jsonpatch "gopkg.in/evanphx/json-patch.v4" + appsv1 "k8s.io/api/apps/v1" + authenticationv1 "k8s.io/api/authentication/v1" + autoscalingv1 "k8s.io/api/autoscaling/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + policyv1beta1 "k8s.io/api/policy/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + utilrand "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/strategicpatch" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/testing" + "k8s.io/utils/ptr" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/internal/field/selector" + "sigs.k8s.io/controller-runtime/pkg/internal/objectutil" +) + +type versionedTracker struct { + testing.ObjectTracker + scheme *runtime.Scheme + withStatusSubresource sets.Set[schema.GroupVersionKind] +} + +type fakeClient struct { + // trackerWriteLock must be acquired before writing to + // the tracker or performing reads that affect a following + // write. + trackerWriteLock sync.Mutex + tracker versionedTracker + + schemeLock sync.RWMutex + scheme *runtime.Scheme + + restMapper meta.RESTMapper + withStatusSubresource sets.Set[schema.GroupVersionKind] + + // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. + // The inner map maps from index name to IndexerFunc. + indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc + // indexesLock must be held when accessing indexes. + indexesLock sync.RWMutex +} + +var _ client.WithWatch = &fakeClient{} + +const ( + maxNameLength = 63 + randomLength = 5 + maxGeneratedNameLength = maxNameLength - randomLength + + subResourceScale = "scale" +) + +// NewFakeClient creates a new fake client for testing. +// You can choose to initialize it with a slice of runtime.Object. +func NewFakeClient(initObjs ...runtime.Object) client.WithWatch { + return NewClientBuilder().WithRuntimeObjects(initObjs...).Build() +} + +// NewClientBuilder returns a new builder to create a fake client. +func NewClientBuilder() *ClientBuilder { + return &ClientBuilder{} +} + +// ClientBuilder builds a fake client. +type ClientBuilder struct { + scheme *runtime.Scheme + restMapper meta.RESTMapper + initObject []client.Object + initLists []client.ObjectList + initRuntimeObjects []runtime.Object + withStatusSubresource []client.Object + objectTracker testing.ObjectTracker + interceptorFuncs *interceptor.Funcs + + // indexes maps each GroupVersionKind (GVK) to the indexes registered for that GVK. + // The inner map maps from index name to IndexerFunc. + indexes map[schema.GroupVersionKind]map[string]client.IndexerFunc +} + +// WithScheme sets this builder's internal scheme. +// If not set, defaults to client-go's global scheme.Scheme. +func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder { + f.scheme = scheme + return f +} + +// WithRESTMapper sets this builder's restMapper. +// The restMapper is directly set as mapper in the Client. This can be used for example +// with a meta.DefaultRESTMapper to provide a static rest mapping. +// If not set, defaults to an empty meta.DefaultRESTMapper. +func (f *ClientBuilder) WithRESTMapper(restMapper meta.RESTMapper) *ClientBuilder { + f.restMapper = restMapper + return f +} + +// WithObjects can be optionally used to initialize this fake client with client.Object(s). +func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder { + f.initObject = append(f.initObject, initObjs...) + return f +} + +// WithLists can be optionally used to initialize this fake client with client.ObjectList(s). +func (f *ClientBuilder) WithLists(initLists ...client.ObjectList) *ClientBuilder { + f.initLists = append(f.initLists, initLists...) + return f +} + +// WithRuntimeObjects can be optionally used to initialize this fake client with runtime.Object(s). +func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *ClientBuilder { + f.initRuntimeObjects = append(f.initRuntimeObjects, initRuntimeObjs...) + return f +} + +// WithObjectTracker can be optionally used to initialize this fake client with testing.ObjectTracker. +func (f *ClientBuilder) WithObjectTracker(ot testing.ObjectTracker) *ClientBuilder { + f.objectTracker = ot + return f +} + +// WithIndex can be optionally used to register an index with name `field` and indexer `extractValue` +// for API objects of the same GroupVersionKind (GVK) as `obj` in the fake client. +// It can be invoked multiple times, both with objects of the same GVK or different ones. +// Invoking WithIndex twice with the same `field` and GVK (via `obj`) arguments will panic. +// WithIndex retrieves the GVK of `obj` using the scheme registered via WithScheme if +// WithScheme was previously invoked, the default scheme otherwise. +func (f *ClientBuilder) WithIndex(obj runtime.Object, field string, extractValue client.IndexerFunc) *ClientBuilder { + objScheme := f.scheme + if objScheme == nil { + objScheme = scheme.Scheme + } + + gvk, err := apiutil.GVKForObject(obj, objScheme) + if err != nil { + panic(err) + } + + // If this is the first index being registered, we initialize the map storing all the indexes. + if f.indexes == nil { + f.indexes = make(map[schema.GroupVersionKind]map[string]client.IndexerFunc) + } + + // If this is the first index being registered for the GroupVersionKind of `obj`, we initialize + // the map storing the indexes for that GroupVersionKind. + if f.indexes[gvk] == nil { + f.indexes[gvk] = make(map[string]client.IndexerFunc) + } + + if _, fieldAlreadyIndexed := f.indexes[gvk][field]; fieldAlreadyIndexed { + panic(fmt.Errorf("indexer conflict: field %s for GroupVersionKind %v is already indexed", + field, gvk)) + } + + f.indexes[gvk][field] = extractValue + + return f +} + +// WithStatusSubresource configures the passed object with a status subresource, which means +// calls to Update and Patch will not alter its status. +func (f *ClientBuilder) WithStatusSubresource(o ...client.Object) *ClientBuilder { + f.withStatusSubresource = append(f.withStatusSubresource, o...) + return f +} + +// WithInterceptorFuncs configures the client methods to be intercepted using the provided interceptor.Funcs. +func (f *ClientBuilder) WithInterceptorFuncs(interceptorFuncs interceptor.Funcs) *ClientBuilder { + f.interceptorFuncs = &interceptorFuncs + return f +} + +// Build builds and returns a new fake client. +func (f *ClientBuilder) Build() client.WithWatch { + if f.scheme == nil { + f.scheme = scheme.Scheme + } + if f.restMapper == nil { + f.restMapper = meta.NewDefaultRESTMapper([]schema.GroupVersion{}) + } + + var tracker versionedTracker + + withStatusSubResource := sets.New(inTreeResourcesWithStatus()...) + for _, o := range f.withStatusSubresource { + gvk, err := apiutil.GVKForObject(o, f.scheme) + if err != nil { + panic(fmt.Errorf("failed to get gvk for object %T: %w", withStatusSubResource, err)) + } + withStatusSubResource.Insert(gvk) + } + + if f.objectTracker == nil { + tracker = versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme, withStatusSubresource: withStatusSubResource} + } else { + tracker = versionedTracker{ObjectTracker: f.objectTracker, scheme: f.scheme, withStatusSubresource: withStatusSubResource} + } + + for _, obj := range f.initObject { + if err := tracker.Add(obj); err != nil { + panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err)) + } + } + for _, obj := range f.initLists { + if err := tracker.Add(obj); err != nil { + panic(fmt.Errorf("failed to add list %v to fake client: %w", obj, err)) + } + } + for _, obj := range f.initRuntimeObjects { + if err := tracker.Add(obj); err != nil { + panic(fmt.Errorf("failed to add runtime object %v to fake client: %w", obj, err)) + } + } + + var result client.WithWatch = &fakeClient{ + tracker: tracker, + scheme: f.scheme, + restMapper: f.restMapper, + indexes: f.indexes, + withStatusSubresource: withStatusSubResource, + } + + if f.interceptorFuncs != nil { + result = interceptor.NewClient(result, *f.interceptorFuncs) + } + + return result +} + +const trackerAddResourceVersion = "999" + +func (t versionedTracker) Add(obj runtime.Object) error { + var objects []runtime.Object + if meta.IsListType(obj) { + var err error + objects, err = meta.ExtractList(obj) + if err != nil { + return err + } + } else { + objects = []runtime.Object{obj} + } + for _, obj := range objects { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetDeletionTimestamp() != nil && len(accessor.GetFinalizers()) == 0 { + return fmt.Errorf("refusing to create obj %s with metadata.deletionTimestamp but no finalizers", accessor.GetName()) + } + if accessor.GetResourceVersion() == "" { + // We use a "magic" value of 999 here because this field + // is parsed as uint and and 0 is already used in Update. + // As we can't go lower, go very high instead so this can + // be recognized + accessor.SetResourceVersion(trackerAddResourceVersion) + } + + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + if err := t.ObjectTracker.Add(obj); err != nil { + return err + } + } + + return nil +} + +func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.CreateOptions) error { + accessor, err := meta.Accessor(obj) + if err != nil { + return fmt.Errorf("failed to get accessor for object: %w", err) + } + if accessor.GetName() == "" { + return apierrors.NewInvalid( + obj.GetObjectKind().GroupVersionKind().GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + if accessor.GetResourceVersion() != "" { + return apierrors.NewBadRequest("resourceVersion can not be set for Create requests") + } + accessor.SetResourceVersion("1") + obj, err = convertFromUnstructuredIfNecessary(t.scheme, obj) + if err != nil { + return err + } + if err := t.ObjectTracker.Create(gvr, obj, ns, opts...); err != nil { + accessor.SetResourceVersion("") + return err + } + + return nil +} + +// convertFromUnstructuredIfNecessary will convert runtime.Unstructured for a GVK that is recognized +// by the schema into the whatever the schema produces with New() for said GVK. +// This is required because the tracker unconditionally saves on manipulations, but its List() implementation +// tries to assign whatever it finds into a ListType it gets from schema.New() - Thus we have to ensure +// we save as the very same type, otherwise subsequent List requests will fail. +func convertFromUnstructuredIfNecessary(s *runtime.Scheme, o runtime.Object) (runtime.Object, error) { + u, isUnstructured := o.(runtime.Unstructured) + if !isUnstructured { + return o, nil + } + gvk := o.GetObjectKind().GroupVersionKind() + if !s.Recognizes(gvk) { + return o, nil + } + + typed, err := s.New(gvk) + if err != nil { + return nil, fmt.Errorf("scheme recognizes %s but failed to produce an object for it: %w", gvk, err) + } + + unstructuredSerialized, err := json.Marshal(u) + if err != nil { + return nil, fmt.Errorf("failed to serialize %T: %w", unstructuredSerialized, err) + } + if err := json.Unmarshal(unstructuredSerialized, typed); err != nil { + return nil, fmt.Errorf("failed to unmarshal the content of %T into %T: %w", u, typed, err) + } + + return typed, nil +} + +func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.UpdateOptions) error { + updateOpts, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + + return t.update(gvr, obj, ns, false, false, updateOpts) +} + +func (t versionedTracker) update(gvr schema.GroupVersionResource, obj runtime.Object, ns string, isStatus, deleting bool, opts metav1.UpdateOptions) error { + obj, err := t.updateObject(gvr, obj, ns, isStatus, deleting, opts.DryRun) + if err != nil { + return err + } + if obj == nil { + return nil + } + + return t.ObjectTracker.Update(gvr, obj, ns, opts) +} + +func (t versionedTracker) Patch(gvr schema.GroupVersionResource, obj runtime.Object, ns string, opts ...metav1.PatchOptions) error { + patchOptions, err := getSingleOrZeroOptions(opts) + if err != nil { + return err + } + + isStatus := false + // We apply patches using a client-go reaction that ends up calling the trackers Patch. As we can't change + // that reaction, we use the callstack to figure out if this originated from the status client. + if bytes.Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeSubResourceClient).statusPatch")) { + isStatus = true + } + + obj, err = t.updateObject(gvr, obj, ns, isStatus, false, patchOptions.DryRun) + if err != nil { + return err + } + if obj == nil { + return nil + } + + return t.ObjectTracker.Patch(gvr, obj, ns, patchOptions) +} + +func (t versionedTracker) updateObject(gvr schema.GroupVersionResource, obj runtime.Object, ns string, isStatus, deleting bool, dryRun []string) (runtime.Object, error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, fmt.Errorf("failed to get accessor for object: %w", err) + } + + if accessor.GetName() == "" { + return nil, apierrors.NewInvalid( + obj.GetObjectKind().GroupVersionKind().GroupKind(), + accessor.GetName(), + field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")}) + } + + gvk, err := apiutil.GVKForObject(obj, t.scheme) + if err != nil { + return nil, err + } + + oldObject, err := t.ObjectTracker.Get(gvr, ns, accessor.GetName()) + if err != nil { + // If the resource is not found and the resource allows create on update, issue a + // create instead. + if apierrors.IsNotFound(err) && allowsCreateOnUpdate(gvk) { + return nil, t.Create(gvr, obj, ns) + } + return nil, err + } + + if t.withStatusSubresource.Has(gvk) { + if isStatus { // copy everything but status and metadata.ResourceVersion from original object + if err := copyStatusFrom(obj, oldObject); err != nil { + return nil, fmt.Errorf("failed to copy non-status field for object with status subresouce: %w", err) + } + passedRV := accessor.GetResourceVersion() + if err := copyFrom(oldObject, obj); err != nil { + return nil, fmt.Errorf("failed to restore non-status fields: %w", err) + } + accessor.SetResourceVersion(passedRV) + } else { // copy status from original object + if err := copyStatusFrom(oldObject, obj); err != nil { + return nil, fmt.Errorf("failed to copy the status for object with status subresource: %w", err) + } + } + } else if isStatus { + return nil, apierrors.NewNotFound(gvr.GroupResource(), accessor.GetName()) + } + + oldAccessor, err := meta.Accessor(oldObject) + if err != nil { + return nil, err + } + + // If the new object does not have the resource version set and it allows unconditional update, + // default it to the resource version of the existing resource + if accessor.GetResourceVersion() == "" { + switch { + case allowsUnconditionalUpdate(gvk): + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + // This is needed because if the patch explicitly sets the RV to null, the client-go reaction we use + // to apply it and whose output we process here will have it unset. It is not clear why the Kubernetes + // apiserver accepts such a patch, but it does so we just copy that behavior. + // Kubernetes apiserver behavior can be checked like this: + // `kubectl patch configmap foo --patch '{"metadata":{"annotations":{"foo":"bar"},"resourceVersion":null}}' -v=9` + case bytes. + Contains(debug.Stack(), []byte("sigs.k8s.io/controller-runtime/pkg/client/fake.(*fakeClient).Patch")): + // We apply patches using a client-go reaction that ends up calling the trackers Update. As we can't change + // that reaction, we use the callstack to figure out if this originated from the "fakeClient.Patch" func. + accessor.SetResourceVersion(oldAccessor.GetResourceVersion()) + } + } + + if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() { + return nil, apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified")) + } + if oldAccessor.GetResourceVersion() == "" { + oldAccessor.SetResourceVersion("0") + } + intResourceVersion, err := strconv.ParseUint(oldAccessor.GetResourceVersion(), 10, 64) + if err != nil { + return nil, fmt.Errorf("can not convert resourceVersion %q to int: %w", oldAccessor.GetResourceVersion(), err) + } + intResourceVersion++ + accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10)) + + if !deleting && !deletionTimestampEqual(accessor, oldAccessor) { + return nil, fmt.Errorf("error: Unable to edit %s: metadata.deletionTimestamp field is immutable", accessor.GetName()) + } + + if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 { + return nil, t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName(), metav1.DeleteOptions{DryRun: dryRun}) + } + return convertFromUnstructuredIfNecessary(t.scheme, obj) +} + +func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + o, err := c.tracker.Get(gvr, key.Namespace, key.Name) + if err != nil { + return err + } + + _, isUnstructured := obj.(runtime.Unstructured) + _, isPartialObject := obj.(*metav1.PartialObjectMetadata) + + if isUnstructured || isPartialObject { + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + ta, err := meta.TypeAccessor(o) + if err != nil { + return err + } + ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + } + + j, err := json.Marshal(o) + if err != nil { + return err + } + zero(obj) + return json.Unmarshal(j, obj) +} + +func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + gvk, err := apiutil.GVKForObject(list, c.scheme) + if err != nil { + return nil, err + } + + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + return c.tracker.Watch(gvr, listOpts.Namespace) +} + +func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + originalKind := gvk.Kind + + gvk.Kind = strings.TrimSuffix(gvk.Kind, "List") + + if _, isUnstructuredList := obj.(runtime.Unstructured); isUnstructuredList && !c.scheme.Recognizes(gvk) { + // We need to register the ListKind with UnstructuredList: + // https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51 + c.schemeLock.RUnlock() + c.schemeLock.Lock() + c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{}) + c.schemeLock.Unlock() + c.schemeLock.RLock() + } + + listOpts := client.ListOptions{} + listOpts.ApplyOptions(opts) + + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + o, err := c.tracker.List(gvr, gvk, listOpts.Namespace) + if err != nil { + return err + } + + if _, isUnstructured := obj.(runtime.Unstructured); isUnstructured { + ta, err := meta.TypeAccessor(o) + if err != nil { + return err + } + ta.SetKind(originalKind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + } + + j, err := json.Marshal(o) + if err != nil { + return err + } + zero(obj) + objCopy := obj.DeepCopyObject().(client.ObjectList) + if err := json.Unmarshal(j, objCopy); err != nil { + return err + } + + if _, isUnstructured := obj.(runtime.Unstructured); isUnstructured { + ta, err := meta.TypeAccessor(obj) + if err != nil { + return err + } + ta.SetKind(originalKind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + } + + objs, err := meta.ExtractList(objCopy) + if err != nil { + return err + } + + if listOpts.LabelSelector == nil && listOpts.FieldSelector == nil { + return meta.SetList(obj, objs) + } + + // If we're here, either a label or field selector are specified (or both), so before we return + // the list we must filter it. If both selectors are set, they are ANDed. + filteredList, err := c.filterList(objs, gvk, listOpts.LabelSelector, listOpts.FieldSelector) + if err != nil { + return err + } + + return meta.SetList(obj, filteredList) +} + +func (c *fakeClient) filterList(list []runtime.Object, gvk schema.GroupVersionKind, ls labels.Selector, fs fields.Selector) ([]runtime.Object, error) { + // Filter the objects with the label selector + filteredList := list + if ls != nil { + objsFilteredByLabel, err := objectutil.FilterWithLabels(list, ls) + if err != nil { + return nil, err + } + filteredList = objsFilteredByLabel + } + + // Filter the result of the previous pass with the field selector + if fs != nil { + objsFilteredByField, err := c.filterWithFields(filteredList, gvk, fs) + if err != nil { + return nil, err + } + filteredList = objsFilteredByField + } + + return filteredList, nil +} + +func (c *fakeClient) filterWithFields(list []runtime.Object, gvk schema.GroupVersionKind, fs fields.Selector) ([]runtime.Object, error) { + requiresExact := selector.RequiresExactMatch(fs) + if !requiresExact { + return nil, fmt.Errorf(`field selector %s is not in one of the two supported forms "key==val" or "key=val"`, fs) + } + + c.indexesLock.RLock() + defer c.indexesLock.RUnlock() + // Field selection is mimicked via indexes, so there's no sane answer this function can give + // if there are no indexes registered for the GroupVersionKind of the objects in the list. + indexes := c.indexes[gvk] + for _, req := range fs.Requirements() { + if len(indexes) == 0 || indexes[req.Field] == nil { + return nil, fmt.Errorf("List on GroupVersionKind %v specifies selector on field %s, but no "+ + "index with name %s has been registered for GroupVersionKind %v", gvk, req.Field, req.Field, gvk) + } + } + + filteredList := make([]runtime.Object, 0, len(list)) + for _, obj := range list { + matches := true + for _, req := range fs.Requirements() { + indexExtractor := indexes[req.Field] + if !c.objMatchesFieldSelector(obj, indexExtractor, req.Value) { + matches = false + break + } + } + if matches { + filteredList = append(filteredList, obj) + } + } + return filteredList, nil +} + +func (c *fakeClient) objMatchesFieldSelector(o runtime.Object, extractIndex client.IndexerFunc, val string) bool { + obj, isClientObject := o.(client.Object) + if !isClientObject { + panic(fmt.Errorf("expected object %v to be of type client.Object, but it's not", o)) + } + + for _, extractedVal := range extractIndex(obj) { + if extractedVal == val { + return true + } + } + + return false +} + +func (c *fakeClient) Scheme() *runtime.Scheme { + return c.scheme +} + +func (c *fakeClient) RESTMapper() meta.RESTMapper { + return c.restMapper +} + +// GroupVersionKindFor returns the GroupVersionKind for the given object. +func (c *fakeClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return apiutil.GVKForObject(obj, c.scheme) +} + +// IsObjectNamespaced returns true if the GroupVersionKind of the object is namespaced. +func (c *fakeClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return apiutil.IsObjectNamespaced(obj, c.scheme, c.restMapper) +} + +func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + createOptions := &client.CreateOptions{} + createOptions.ApplyOptions(opts) + + for _, dryRunOpt := range createOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + + if accessor.GetName() == "" && accessor.GetGenerateName() != "" { + base := accessor.GetGenerateName() + if len(base) > maxGeneratedNameLength { + base = base[:maxGeneratedNameLength] + } + accessor.SetName(fmt.Sprintf("%s%s", base, utilrand.String(randomLength))) + } + // Ignore attempts to set deletion timestamp + if !accessor.GetDeletionTimestamp().IsZero() { + accessor.SetDeletionTimestamp(nil) + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + return c.tracker.Create(gvr, obj, accessor.GetNamespace()) +} + +func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + delOptions := client.DeleteOptions{} + delOptions.ApplyOptions(opts) + + for _, dryRunOpt := range delOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + // Check the ResourceVersion if that Precondition was specified. + if delOptions.Preconditions != nil && delOptions.Preconditions.ResourceVersion != nil { + name := accessor.GetName() + dbObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), name) + if err != nil { + return err + } + oldAccessor, err := meta.Accessor(dbObj) + if err != nil { + return err + } + actualRV := oldAccessor.GetResourceVersion() + expectRV := *delOptions.Preconditions.ResourceVersion + if actualRV != expectRV { + msg := fmt.Sprintf( + "the ResourceVersion in the precondition (%s) does not match the ResourceVersion in record (%s). "+ + "The object might have been modified", + expectRV, actualRV) + return apierrors.NewConflict(gvr.GroupResource(), name, errors.New(msg)) + } + } + + return c.deleteObjectLocked(gvr, accessor) +} + +func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + dcOptions := client.DeleteAllOfOptions{} + dcOptions.ApplyOptions(opts) + + for _, dryRunOpt := range dcOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + o, err := c.tracker.List(gvr, gvk, dcOptions.Namespace) + if err != nil { + return err + } + + objs, err := meta.ExtractList(o) + if err != nil { + return err + } + filteredObjs, err := objectutil.FilterWithLabels(objs, dcOptions.LabelSelector) + if err != nil { + return err + } + for _, o := range filteredObjs { + accessor, err := meta.Accessor(o) + if err != nil { + return err + } + err = c.deleteObjectLocked(gvr, accessor) + if err != nil { + return err + } + } + return nil +} + +func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + return c.update(obj, false, opts...) +} + +func (c *fakeClient) update(obj client.Object, isStatus bool, opts ...client.UpdateOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + updateOptions := &client.UpdateOptions{} + updateOptions.ApplyOptions(opts) + + for _, dryRunOpt := range updateOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + return c.tracker.update(gvr, obj, accessor.GetNamespace(), isStatus, false, *updateOptions.AsUpdateOptions()) +} + +func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return c.patch(obj, patch, opts...) +} + +func (c *fakeClient) patch(obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + c.schemeLock.RLock() + defer c.schemeLock.RUnlock() + patchOptions := &client.PatchOptions{} + patchOptions.ApplyOptions(opts) + + for _, dryRunOpt := range patchOptions.DryRun { + if dryRunOpt == metav1.DryRunAll { + return nil + } + } + + gvr, err := getGVRFromObject(obj, c.scheme) + if err != nil { + return err + } + accessor, err := meta.Accessor(obj) + if err != nil { + return err + } + data, err := patch.Data(obj) + if err != nil { + return err + } + + gvk, err := apiutil.GVKForObject(obj, c.scheme) + if err != nil { + return err + } + + c.trackerWriteLock.Lock() + defer c.trackerWriteLock.Unlock() + oldObj, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) + if err != nil { + return err + } + oldAccessor, err := meta.Accessor(oldObj) + if err != nil { + return err + } + + // Apply patch without updating object. + // To remain in accordance with the behavior of k8s api behavior, + // a patch must not allow for changes to the deletionTimestamp of an object. + // The reaction() function applies the patch to the object and calls Update(), + // whereas dryPatch() replicates this behavior but skips the call to Update(). + // This ensures that the patch may be rejected if a deletionTimestamp is modified, prior + // to updating the object. + action := testing.NewPatchAction(gvr, accessor.GetNamespace(), accessor.GetName(), patch.Type(), data) + o, err := dryPatch(action, c.tracker) + if err != nil { + return err + } + newObj, err := meta.Accessor(o) + if err != nil { + return err + } + + // Validate that deletionTimestamp has not been changed + if !deletionTimestampEqual(newObj, oldAccessor) { + return fmt.Errorf("rejected patch, metadata.deletionTimestamp immutable") + } + + reaction := testing.ObjectReaction(c.tracker) + handled, o, err := reaction(action) + if err != nil { + return err + } + if !handled { + panic("tracker could not handle patch method") + } + + if _, isUnstructured := obj.(runtime.Unstructured); isUnstructured { + ta, err := meta.TypeAccessor(o) + if err != nil { + return err + } + ta.SetKind(gvk.Kind) + ta.SetAPIVersion(gvk.GroupVersion().String()) + } + + j, err := json.Marshal(o) + if err != nil { + return err + } + zero(obj) + return json.Unmarshal(j, obj) +} + +// Applying a patch results in a deletionTimestamp that is truncated to the nearest second. +// Check that the diff between a new and old deletion timestamp is within a reasonable threshold +// to be considered unchanged. +func deletionTimestampEqual(newObj metav1.Object, obj metav1.Object) bool { + newTime := newObj.GetDeletionTimestamp() + oldTime := obj.GetDeletionTimestamp() + + if newTime == nil || oldTime == nil { + return newTime == oldTime + } + return newTime.Time.Sub(oldTime.Time).Abs() < time.Second +} + +// The behavior of applying the patch is pulled out into dryPatch(), +// which applies the patch and returns an object, but does not Update() the object. +// This function returns a patched runtime object that may then be validated before a call to Update() is executed. +// This results in some code duplication, but was found to be a cleaner alternative than unmarshalling and introspecting the patch data +// and easier than refactoring the k8s client-go method upstream. +// Duplicate of upstream: https://github.com/kubernetes/client-go/blob/783d0d33626e59d55d52bfd7696b775851f92107/testing/fixture.go#L146-L194 +func dryPatch(action testing.PatchActionImpl, tracker testing.ObjectTracker) (runtime.Object, error) { + ns := action.GetNamespace() + gvr := action.GetResource() + + obj, err := tracker.Get(gvr, ns, action.GetName()) + if err != nil { + return nil, err + } + + old, err := json.Marshal(obj) + if err != nil { + return nil, err + } + + // reset the object in preparation to unmarshal, since unmarshal does not guarantee that fields + // in obj that are removed by patch are cleared + value := reflect.ValueOf(obj) + value.Elem().Set(reflect.New(value.Type().Elem()).Elem()) + + switch action.GetPatchType() { + case types.JSONPatchType: + patch, err := jsonpatch.DecodePatch(action.GetPatch()) + if err != nil { + return nil, err + } + modified, err := patch.Apply(old) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(modified, obj); err != nil { + return nil, err + } + case types.MergePatchType: + modified, err := jsonpatch.MergePatch(old, action.GetPatch()) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(modified, obj); err != nil { + return nil, err + } + case types.StrategicMergePatchType: + mergedByte, err := strategicpatch.StrategicMergePatch(old, action.GetPatch(), obj) + if err != nil { + return nil, err + } + if err = json.Unmarshal(mergedByte, obj); err != nil { + return nil, err + } + case types.ApplyPatchType: + return nil, errors.New("apply patches are not supported in the fake client. Follow https://github.com/kubernetes/kubernetes/issues/115598 for the current status") + case types.ApplyCBORPatchType: + return nil, errors.New("apply CBOR patches are not supported in the fake client") + default: + return nil, fmt.Errorf("%s PatchType is not supported", action.GetPatchType()) + } + return obj, nil +} + +// copyStatusFrom copies the status from old into new +func copyStatusFrom(old, new runtime.Object) error { + oldMapStringAny, err := toMapStringAny(old) + if err != nil { + return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err) + } + newMapStringAny, err := toMapStringAny(new) + if err != nil { + return fmt.Errorf("failed to convert new to *unststructured.Unstructured: %w", err) + } + + newMapStringAny["status"] = oldMapStringAny["status"] + + if err := fromMapStringAny(newMapStringAny, new); err != nil { + return fmt.Errorf("failed to convert back from map[string]any: %w", err) + } + + return nil +} + +// copyFrom copies from old into new +func copyFrom(old, new runtime.Object) error { + oldMapStringAny, err := toMapStringAny(old) + if err != nil { + return fmt.Errorf("failed to convert old to *unstructured.Unstructured: %w", err) + } + if err := fromMapStringAny(oldMapStringAny, new); err != nil { + return fmt.Errorf("failed to convert back from map[string]any: %w", err) + } + + return nil +} + +func toMapStringAny(obj runtime.Object) (map[string]any, error) { + if unstructured, isUnstructured := obj.(*unstructured.Unstructured); isUnstructured { + return unstructured.Object, nil + } + + serialized, err := json.Marshal(obj) + if err != nil { + return nil, err + } + + u := map[string]any{} + return u, json.Unmarshal(serialized, &u) +} + +func fromMapStringAny(u map[string]any, target runtime.Object) error { + if targetUnstructured, isUnstructured := target.(*unstructured.Unstructured); isUnstructured { + targetUnstructured.Object = u + return nil + } + + serialized, err := json.Marshal(u) + if err != nil { + return fmt.Errorf("failed to serialize: %w", err) + } + + zero(target) + if err := json.Unmarshal(serialized, &target); err != nil { + return fmt.Errorf("failed to deserialize: %w", err) + } + + return nil +} + +func (c *fakeClient) Status() client.SubResourceWriter { + return c.SubResource("status") +} + +func (c *fakeClient) SubResource(subResource string) client.SubResourceClient { + return &fakeSubResourceClient{client: c, subResource: subResource} +} + +func (c *fakeClient) deleteObjectLocked(gvr schema.GroupVersionResource, accessor metav1.Object) error { + old, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName()) + if err == nil { + oldAccessor, err := meta.Accessor(old) + if err == nil { + if len(oldAccessor.GetFinalizers()) > 0 { + now := metav1.Now() + oldAccessor.SetDeletionTimestamp(&now) + // Call update directly with mutability parameter set to true to allow + // changes to deletionTimestamp + return c.tracker.update(gvr, old, accessor.GetNamespace(), false, true, metav1.UpdateOptions{}) + } + } + } + + //TODO: implement propagation + return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName()) +} + +func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionResource, error) { + gvk, err := apiutil.GVKForObject(obj, scheme) + if err != nil { + return schema.GroupVersionResource{}, err + } + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + return gvr, nil +} + +type fakeSubResourceClient struct { + client *fakeClient + subResource string +} + +func (sw *fakeSubResourceClient) Get(ctx context.Context, obj, subResource client.Object, opts ...client.SubResourceGetOption) error { + switch sw.subResource { + case subResourceScale: + // Actual client looks up resource, then extracts the scale sub-resource: + // https://github.com/kubernetes/kubernetes/blob/fb6bbc9781d11a87688c398778525c4e1dcb0f08/pkg/registry/apps/deployment/storage/storage.go#L307 + if err := sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj); err != nil { + return err + } + scale, isScale := subResource.(*autoscalingv1.Scale) + if !isScale { + return apierrors.NewBadRequest(fmt.Sprintf("expected Scale, got %T", subResource)) + } + scaleOut, err := extractScale(obj) + if err != nil { + return err + } + *scale = *scaleOut + return nil + default: + return fmt.Errorf("fakeSubResourceClient does not support get for %s", sw.subResource) + } +} + +func (sw *fakeSubResourceClient) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + switch sw.subResource { + case "eviction": + _, isEviction := subResource.(*policyv1beta1.Eviction) + if !isEviction { + _, isEviction = subResource.(*policyv1.Eviction) + } + if !isEviction { + return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %T, expected Eviction", subResource)) + } + if _, isPod := obj.(*corev1.Pod); !isPod { + return apierrors.NewNotFound(schema.GroupResource{}, "") + } + + return sw.client.Delete(ctx, obj) + case "token": + tokenRequest, isTokenRequest := subResource.(*authenticationv1.TokenRequest) + if !isTokenRequest { + return apierrors.NewBadRequest(fmt.Sprintf("got invalid type %T, expected TokenRequest", subResource)) + } + if _, isServiceAccount := obj.(*corev1.ServiceAccount); !isServiceAccount { + return apierrors.NewNotFound(schema.GroupResource{}, "") + } + + tokenRequest.Status.Token = "fake-token" + tokenRequest.Status.ExpirationTimestamp = metav1.Date(6041, 1, 1, 0, 0, 0, 0, time.UTC) + + return sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj) + default: + return fmt.Errorf("fakeSubResourceWriter does not support create for %s", sw.subResource) + } +} + +func (sw *fakeSubResourceClient) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + updateOptions := client.SubResourceUpdateOptions{} + updateOptions.ApplyOptions(opts) + + switch sw.subResource { + case subResourceScale: + if err := sw.client.Get(ctx, client.ObjectKeyFromObject(obj), obj.DeepCopyObject().(client.Object)); err != nil { + return err + } + if updateOptions.SubResourceBody == nil { + return apierrors.NewBadRequest("missing SubResourceBody") + } + + scale, isScale := updateOptions.SubResourceBody.(*autoscalingv1.Scale) + if !isScale { + return apierrors.NewBadRequest(fmt.Sprintf("expected Scale, got %T", updateOptions.SubResourceBody)) + } + if err := applyScale(obj, scale); err != nil { + return err + } + return sw.client.update(obj, false, &updateOptions.UpdateOptions) + default: + body := obj + if updateOptions.SubResourceBody != nil { + body = updateOptions.SubResourceBody + } + return sw.client.update(body, true, &updateOptions.UpdateOptions) + } +} + +func (sw *fakeSubResourceClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + patchOptions := client.SubResourcePatchOptions{} + patchOptions.ApplyOptions(opts) + + body := obj + if patchOptions.SubResourceBody != nil { + body = patchOptions.SubResourceBody + } + + // this is necessary to identify that last call was made for status patch, through stack trace. + if sw.subResource == "status" { + return sw.statusPatch(body, patch, patchOptions) + } + + return sw.client.patch(body, patch, &patchOptions.PatchOptions) +} + +func (sw *fakeSubResourceClient) statusPatch(body client.Object, patch client.Patch, patchOptions client.SubResourcePatchOptions) error { + return sw.client.patch(body, patch, &patchOptions.PatchOptions) +} + +func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool { + switch gvk.Group { + case "apps": + switch gvk.Kind { + case "ControllerRevision", "DaemonSet", "Deployment", "ReplicaSet", "StatefulSet": + return true + } + case "autoscaling": + switch gvk.Kind { + case "HorizontalPodAutoscaler": + return true + } + case "batch": + switch gvk.Kind { + case "CronJob", "Job": + return true + } + case "certificates": + switch gvk.Kind { + case "Certificates": + return true + } + case "flowcontrol": + switch gvk.Kind { + case "FlowSchema", "PriorityLevelConfiguration": + return true + } + case "networking": + switch gvk.Kind { + case "Ingress", "IngressClass", "NetworkPolicy": + return true + } + case "policy": + switch gvk.Kind { + case "PodSecurityPolicy": + return true + } + case "rbac.authorization.k8s.io": + switch gvk.Kind { + case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding": + return true + } + case "scheduling": + switch gvk.Kind { + case "PriorityClass": + return true + } + case "settings": + switch gvk.Kind { + case "PodPreset": + return true + } + case "storage": + switch gvk.Kind { + case "StorageClass": + return true + } + case "": + switch gvk.Kind { + case "ConfigMap", "Endpoint", "Event", "LimitRange", "Namespace", "Node", + "PersistentVolume", "PersistentVolumeClaim", "Pod", "PodTemplate", + "ReplicationController", "ResourceQuota", "Secret", "Service", + "ServiceAccount", "EndpointSlice": + return true + } + } + + return false +} + +func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool { + switch gvk.Group { + case "coordination": + switch gvk.Kind { + case "Lease": + return true + } + case "node": + switch gvk.Kind { + case "RuntimeClass": + return true + } + case "rbac": + switch gvk.Kind { + case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding": + return true + } + case "": + switch gvk.Kind { + case "Endpoint", "Event", "LimitRange", "Service": + return true + } + } + + return false +} + +func inTreeResourcesWithStatus() []schema.GroupVersionKind { + return []schema.GroupVersionKind{ + {Version: "v1", Kind: "Namespace"}, + {Version: "v1", Kind: "Node"}, + {Version: "v1", Kind: "PersistentVolumeClaim"}, + {Version: "v1", Kind: "PersistentVolume"}, + {Version: "v1", Kind: "Pod"}, + {Version: "v1", Kind: "ReplicationController"}, + {Version: "v1", Kind: "Service"}, + + {Group: "apps", Version: "v1", Kind: "Deployment"}, + {Group: "apps", Version: "v1", Kind: "DaemonSet"}, + {Group: "apps", Version: "v1", Kind: "ReplicaSet"}, + {Group: "apps", Version: "v1", Kind: "StatefulSet"}, + + {Group: "autoscaling", Version: "v1", Kind: "HorizontalPodAutoscaler"}, + + {Group: "batch", Version: "v1", Kind: "CronJob"}, + {Group: "batch", Version: "v1", Kind: "Job"}, + + {Group: "certificates.k8s.io", Version: "v1", Kind: "CertificateSigningRequest"}, + + {Group: "networking.k8s.io", Version: "v1", Kind: "Ingress"}, + {Group: "networking.k8s.io", Version: "v1", Kind: "NetworkPolicy"}, + + {Group: "policy", Version: "v1", Kind: "PodDisruptionBudget"}, + + {Group: "storage.k8s.io", Version: "v1", Kind: "VolumeAttachment"}, + + {Group: "apiextensions.k8s.io", Version: "v1", Kind: "CustomResourceDefinition"}, + + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "FlowSchema"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1beta2", Kind: "PriorityLevelConfiguration"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1", Kind: "FlowSchema"}, + {Group: "flowcontrol.apiserver.k8s.io", Version: "v1", Kind: "PriorityLevelConfiguration"}, + } +} + +// zero zeros the value of a pointer. +func zero(x interface{}) { + if x == nil { + return + } + res := reflect.ValueOf(x).Elem() + res.Set(reflect.Zero(res.Type())) +} + +// getSingleOrZeroOptions returns the single options value in the slice, its +// zero value if the slice is empty, or an error if the slice contains more than +// one option value. +func getSingleOrZeroOptions[T any](opts []T) (opt T, err error) { + switch len(opts) { + case 0: + case 1: + opt = opts[0] + default: + err = fmt.Errorf("expected single or no options value, got %d values", len(opts)) + } + return +} + +func extractScale(obj client.Object) (*autoscalingv1.Scale, error) { + switch obj := obj.(type) { + case *appsv1.Deployment: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + case *appsv1.ReplicaSet: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + case *corev1.ReplicationController: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: labels.Set(obj.Spec.Selector).String(), + }, + }, nil + case *appsv1.StatefulSet: + var replicas int32 = 1 + if obj.Spec.Replicas != nil { + replicas = *obj.Spec.Replicas + } + var selector string + if obj.Spec.Selector != nil { + selector = obj.Spec.Selector.String() + } + return &autoscalingv1.Scale{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: obj.Namespace, + Name: obj.Name, + UID: obj.UID, + ResourceVersion: obj.ResourceVersion, + CreationTimestamp: obj.CreationTimestamp, + }, + Spec: autoscalingv1.ScaleSpec{ + Replicas: replicas, + }, + Status: autoscalingv1.ScaleStatus{ + Replicas: obj.Status.Replicas, + Selector: selector, + }, + }, nil + default: + // TODO: CRDs https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource + return nil, fmt.Errorf("unimplemented scale subresource for resource %T", obj) + } +} + +func applyScale(obj client.Object, scale *autoscalingv1.Scale) error { + switch obj := obj.(type) { + case *appsv1.Deployment: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *appsv1.ReplicaSet: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *corev1.ReplicationController: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + case *appsv1.StatefulSet: + obj.Spec.Replicas = ptr.To(scale.Spec.Replicas) + default: + // TODO: CRDs https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource + return fmt.Errorf("unimplemented scale subresource for resource %T", obj) + } + return nil +} + +// AddIndex adds an index to a fake client. It will panic if used with a client that is not a fake client. +// It will error if there is already an index for given object with the same name as field. +// +// It can be used to test code that adds indexes to the cache at runtime. +func AddIndex(c client.Client, obj runtime.Object, field string, extractValue client.IndexerFunc) error { + fakeClient, isFakeClient := c.(*fakeClient) + if !isFakeClient { + panic("AddIndex can only be used with a fake client") + } + fakeClient.indexesLock.Lock() + defer fakeClient.indexesLock.Unlock() + + if fakeClient.indexes == nil { + fakeClient.indexes = make(map[schema.GroupVersionKind]map[string]client.IndexerFunc, 1) + } + + gvk, err := apiutil.GVKForObject(obj, fakeClient.scheme) + if err != nil { + return fmt.Errorf("failed to get gvk for %T: %w", obj, err) + } + + if fakeClient.indexes[gvk] == nil { + fakeClient.indexes[gvk] = make(map[string]client.IndexerFunc, 1) + } + + if fakeClient.indexes[gvk][field] != nil { + return fmt.Errorf("index %s already exists", field) + } + + fakeClient.indexes[gvk][field] = extractValue + + return nil +} diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go new file mode 100644 index 000000000..47cad3980 --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/fake/doc.go @@ -0,0 +1,38 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package fake provides a fake client for testing. + +A fake client is backed by its simple object store indexed by GroupVersionResource. +You can create a fake client with optional objects. + + client := NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build() + +You can invoke the methods defined in the Client interface. + +When in doubt, it's almost always better not to use this package and instead use +envtest.Environment with a real client and API server. + +WARNING: ⚠️ Current Limitations / Known Issues with the fake Client ⚠️ + - This client does not have a way to inject specific errors to test handled vs. unhandled errors. + - There is some support for sub resources which can cause issues with tests if you're trying to update + e.g. metadata and status in the same reconcile. + - No OpenAPI validation is performed when creating or updating objects. + - ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update + operations that rely on these fields will fail, or give false positives. +*/ +package fake diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/client/interceptor/intercept.go b/vendor/sigs.k8s.io/controller-runtime/pkg/client/interceptor/intercept.go new file mode 100644 index 000000000..3d3f3cb01 --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/client/interceptor/intercept.go @@ -0,0 +1,166 @@ +package interceptor + +import ( + "context" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/watch" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Funcs contains functions that are called instead of the underlying client's methods. +type Funcs struct { + Get func(ctx context.Context, client client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error + List func(ctx context.Context, client client.WithWatch, list client.ObjectList, opts ...client.ListOption) error + Create func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.CreateOption) error + Delete func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteOption) error + DeleteAllOf func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.DeleteAllOfOption) error + Update func(ctx context.Context, client client.WithWatch, obj client.Object, opts ...client.UpdateOption) error + Patch func(ctx context.Context, client client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error + Watch func(ctx context.Context, client client.WithWatch, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) + SubResource func(client client.WithWatch, subResource string) client.SubResourceClient + SubResourceGet func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error + SubResourceCreate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error + SubResourceUpdate func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, opts ...client.SubResourceUpdateOption) error + SubResourcePatch func(ctx context.Context, client client.Client, subResourceName string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error +} + +// NewClient returns a new interceptor client that calls the functions in funcs instead of the underlying client's methods, if they are not nil. +func NewClient(interceptedClient client.WithWatch, funcs Funcs) client.WithWatch { + return interceptor{ + client: interceptedClient, + funcs: funcs, + } +} + +type interceptor struct { + client client.WithWatch + funcs Funcs +} + +var _ client.WithWatch = &interceptor{} + +func (c interceptor) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + return c.client.GroupVersionKindFor(obj) +} + +func (c interceptor) IsObjectNamespaced(obj runtime.Object) (bool, error) { + return c.client.IsObjectNamespaced(obj) +} + +func (c interceptor) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if c.funcs.Get != nil { + return c.funcs.Get(ctx, c.client, key, obj, opts...) + } + return c.client.Get(ctx, key, obj, opts...) +} + +func (c interceptor) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if c.funcs.List != nil { + return c.funcs.List(ctx, c.client, list, opts...) + } + return c.client.List(ctx, list, opts...) +} + +func (c interceptor) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if c.funcs.Create != nil { + return c.funcs.Create(ctx, c.client, obj, opts...) + } + return c.client.Create(ctx, obj, opts...) +} + +func (c interceptor) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + if c.funcs.Delete != nil { + return c.funcs.Delete(ctx, c.client, obj, opts...) + } + return c.client.Delete(ctx, obj, opts...) +} + +func (c interceptor) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if c.funcs.Update != nil { + return c.funcs.Update(ctx, c.client, obj, opts...) + } + return c.client.Update(ctx, obj, opts...) +} + +func (c interceptor) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if c.funcs.Patch != nil { + return c.funcs.Patch(ctx, c.client, obj, patch, opts...) + } + return c.client.Patch(ctx, obj, patch, opts...) +} + +func (c interceptor) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + if c.funcs.DeleteAllOf != nil { + return c.funcs.DeleteAllOf(ctx, c.client, obj, opts...) + } + return c.client.DeleteAllOf(ctx, obj, opts...) +} + +func (c interceptor) Status() client.SubResourceWriter { + return c.SubResource("status") +} + +func (c interceptor) SubResource(subResource string) client.SubResourceClient { + if c.funcs.SubResource != nil { + return c.funcs.SubResource(c.client, subResource) + } + return subResourceInterceptor{ + subResourceName: subResource, + client: c.client, + funcs: c.funcs, + } +} + +func (c interceptor) Scheme() *runtime.Scheme { + return c.client.Scheme() +} + +func (c interceptor) RESTMapper() meta.RESTMapper { + return c.client.RESTMapper() +} + +func (c interceptor) Watch(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) (watch.Interface, error) { + if c.funcs.Watch != nil { + return c.funcs.Watch(ctx, c.client, obj, opts...) + } + return c.client.Watch(ctx, obj, opts...) +} + +type subResourceInterceptor struct { + subResourceName string + client client.Client + funcs Funcs +} + +var _ client.SubResourceClient = &subResourceInterceptor{} + +func (s subResourceInterceptor) Get(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceGetOption) error { + if s.funcs.SubResourceGet != nil { + return s.funcs.SubResourceGet(ctx, s.client, s.subResourceName, obj, subResource, opts...) + } + return s.client.SubResource(s.subResourceName).Get(ctx, obj, subResource, opts...) +} + +func (s subResourceInterceptor) Create(ctx context.Context, obj client.Object, subResource client.Object, opts ...client.SubResourceCreateOption) error { + if s.funcs.SubResourceCreate != nil { + return s.funcs.SubResourceCreate(ctx, s.client, s.subResourceName, obj, subResource, opts...) + } + return s.client.SubResource(s.subResourceName).Create(ctx, obj, subResource, opts...) +} + +func (s subResourceInterceptor) Update(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if s.funcs.SubResourceUpdate != nil { + return s.funcs.SubResourceUpdate(ctx, s.client, s.subResourceName, obj, opts...) + } + return s.client.SubResource(s.subResourceName).Update(ctx, obj, opts...) +} + +func (s subResourceInterceptor) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + if s.funcs.SubResourcePatch != nil { + return s.funcs.SubResourcePatch(ctx, s.client, s.subResourceName, obj, patch, opts...) + } + return s.client.SubResource(s.subResourceName).Patch(ctx, obj, patch, opts...) +} diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/internal/objectutil/objectutil.go b/vendor/sigs.k8s.io/controller-runtime/pkg/internal/objectutil/objectutil.go new file mode 100644 index 000000000..0189c0432 --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/internal/objectutil/objectutil.go @@ -0,0 +1,42 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package objectutil + +import ( + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" +) + +// FilterWithLabels returns a copy of the items in objs matching labelSel. +func FilterWithLabels(objs []runtime.Object, labelSel labels.Selector) ([]runtime.Object, error) { + outItems := make([]runtime.Object, 0, len(objs)) + for _, obj := range objs { + meta, err := apimeta.Accessor(obj) + if err != nil { + return nil, err + } + if labelSel != nil { + lbls := labels.Set(meta.GetLabels()) + if !labelSel.Matches(lbls) { + continue + } + } + outItems = append(outItems, obj.DeepCopyObject()) + } + return outItems, nil +}