feat(ha): drift backstop and failure events for the state mirror

Dockerfile

@@ -23,6 +23,7 @@ COPY util/ util/
 COPY events/ events/
 COPY metrics/ metrics/
 COPY cleanup/ cleanup/
+COPY pkg/ pkg/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o manager main.go

config/events/controller.yaml

@@ -539,3 +539,9 @@ events:
     type: Warning
     reportingController: controller
     message: Warning - Certificate Creation job Failed
+  - name: HAMirrorSyncFailed
+    reason: HAMirrorSyncFailed
+    action: HAMirrorSync
+    type: Warning
+    reportingController: controller
+    message: Failed to mirror a resource from the Active hub onto the Standby; the syncer will retry.

config/events/events_config_map.yaml

@@ -96,4 +96,5 @@ data:
       - CertificateJobCreationFailed
       - CertificatesRenewNow
       - IllegalVPNKeyRotationConfigDelete
-      - CertificateJobFailed
+      - CertificateJobFailed
+      - HAMirrorSyncFailed

config/ha/README.md

@@ -0,0 +1,48 @@
+# HA cross-cluster RBAC (issue #295)
+
+`active-cluster-clusterrole.yaml` is a least-privilege grant for the
+identity behind a Standby's `--ha-active-kubeconfig` flag: read-only
+(`get`/`list`/`watch`) access to `Namespace` plus every resource type in
+`pkg/ha.CRDMirrorSet` (`RemoteSyncer` never writes to the Active cluster —
+only reads), plus the Active's own `coordination.k8s.io/v1` `Lease` —
+the same kubeconfig is also used by #294's `WatchRemoteLease` to read the
+Active's Lease directly, not just by `RemoteSyncer` to mirror CRDs.
+
+## This is not applied by this repo's own deploy flow
+
+Nothing here is referenced by `config/rbac/kustomization.yaml` or
+`config/default/kustomization.yaml`. Those govern the RBAC a controller
+grants *itself* on the cluster it's running in. This manifest is different:
+it must be applied **on the Active hub cluster**, granting access to
+whatever identity the *Standby's* kubeconfig authenticates as — a cluster
+this repo's own kustomize overlays have no way to reach, since it's a
+separate cluster entirely.
+
+Apply it manually (or via whatever provisioning tooling manages the Active
+hub) against the Active cluster:
+
+```
+kubectl --context <active-hub-context> apply -f active-cluster-clusterrole.yaml
+```
+
+Fill in the `ClusterRoleBinding`'s `subjects` first — the correct subject
+depends on how the Standby authenticates to the Active (a `ServiceAccount`
+if dialing in-cluster, a client-cert `User` if using a flattened
+kubeconfig Secret, as the current dev demo does).
+
+## A known, deliberate gap this manifest does not close
+
+Nothing in this repo automates applying this to a real Active cluster —
+that's cross-cluster provisioning, out of scope for a single controller
+repo. Logged as a follow-up, not built.
+
+## Credential mirroring (a later PR)
+
+If/when `pkg/ha.CredentialMirrorSet` (Secrets, ServiceAccounts, Roles,
+RoleBindings — for #297's post-promotion use) is wired in, this
+`ClusterRole` will need `secrets`/`serviceaccounts`/`roles`/`rolebindings`
+appended. Worth knowing ahead of time: RBAC cannot scope `Secret` access by
+`.type`, so that addition grants read access to **every** Secret in the
+project namespaces on the Active cluster, not just the credential ones
+`RemoteSyncer` actually mirrors — a real credential-exposure tradeoff to
+weigh when that lands, not just an implementation detail.

config/ha/active-cluster-clusterrole.yaml

@@ -0,0 +1,78 @@
+---
+# Least-privilege grant for the identity behind the Standby's
+# --ha-active-kubeconfig, applied ON THE ACTIVE CLUSTER — not part of this
+# repo's own config/rbac (that governs the local cluster's own ClusterRole)
+# and not referenced by config/rbac/kustomization.yaml or
+# config/default/kustomization.yaml, so it is never auto-applied to the
+# Standby's own cluster, where it would be meaningless.
+#
+# See config/ha/README.md for how and where to apply this.
+#
+# Scope: read-only (get/list/watch) on Namespace plus every type in
+# pkg/ha.CRDMirrorSet, plus the Active's own HA Lease — the same
+# --ha-active-kubeconfig identity is also used by #294's WatchRemoteLease to
+# read the Active's coordination.k8s.io/v1 Lease (checkRemoteLeaseOnce),
+# not just by RemoteSyncer. Confirmed live: without this, the Standby can
+# mirror CRDs fine but permanently fails to read the Active's Lease,
+# so it can never observe staleness in the first place.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeslice-ha-standby-reader
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - controller.kubeslice.io
+    resources:
+      - projects
+      - clusters
+      - sliceconfigs
+      - serviceexportconfigs
+      - sliceqosconfigs
+      - vpnkeyrotations
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - worker.kubeslice.io
+    resources:
+      - workersliceconfigs
+      - workerslicegateways
+      - workerserviceimports
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Template only — the subject is deployment-specific (a ServiceAccount if the
+# Standby dials the Active in-cluster, a cert CN if it uses client-cert auth
+# via a flattened kubeconfig) and can't be hardcoded here. Fill in `subjects`
+# before applying.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeslice-ha-standby-reader-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeslice-ha-standby-reader
+subjects:
+  - kind: User # or ServiceAccount — see README.md
+    name: CHANGEME
+    apiGroup: rbac.authorization.k8s.io

config/rbac/role.yaml

@@ -30,6 +30,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - ""
   resources:

controllers/controller/cluster_controller.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -36,6 +38,9 @@ type ClusterReconciler struct {
 	ClusterService service.IClusterService
 	Log            *zap.SugaredLogger
 	EventRecorder  *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -47,6 +52,12 @@ func (c *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the cluster , ClusterReconciler implements it
 func (c *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if c.LeaderElector != nil && !c.LeaderElector.IsLeader() {
+		c.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, c.Client, c.Scheme, "ClusterController", c.EventRecorder)
 	return c.ClusterService.ReconcileCluster(kubeSliceCtx, req)
 }

controllers/controller/leader_gate_test.go

@@ -0,0 +1,62 @@
+/*
+ * 	Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0
+ *
+ * 	Licensed under the Apache License, Version 2.0 (the "License");
+ * 	you may not use this file except in compliance with the License.
+ * 	You may obtain a copy of the License at
+ *
+ * 	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * 	Unless required by applicable law or agreed to in writing, software
+ * 	distributed under the License is distributed on an "AS IS" BASIS,
+ * 	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * 	See the License for the specific language governing permissions and
+ * 	limitations under the License.
+ */
+
+package controller
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"go.uber.org/zap/zaptest/observer"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
+)
+
+// TestReconcile_StandbySkipsAndLogs verifies the HA write fence: a Standby
+// controller returns immediately from Reconcile without touching its service
+// (left nil here — a leaking gate would panic), and logs the skip message on
+// every call, proving IsLeader() is evaluated per invocation.
+func TestReconcile_StandbySkipsAndLogs(t *testing.T) {
+	core, logs := observer.New(zapcore.InfoLevel)
+	logger := zap.New(core).Sugar()
+
+	standby := ha.NewClusterLeaderElector(nil, nil, ha.Options{
+		Mode: ha.ModeStandby,
+		Log:  zap.NewNop().Sugar(),
+	})
+	require.False(t, standby.IsLeader(), "standby must not be leader")
+
+	r := &SliceConfigReconciler{
+		Log:           logger,
+		LeaderElector: standby,
+		// SliceConfigService is intentionally nil: the gate must return before it.
+	}
+
+	const calls = 3
+	for i := 0; i < calls; i++ {
+		res, err := r.Reconcile(context.Background(), ctrl.Request{})
+		require.NoError(t, err)
+		assert.Equal(t, ctrl.Result{}, res)
+	}
+
+	assert.Equal(t, calls, logs.FilterMessage("standby mode, skipping reconcile").Len(),
+		"expected one skip log per Reconcile call")
+}

controllers/controller/project_controller.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -36,6 +38,9 @@ type ProjectReconciler struct {
 	ProjectService service.IProjectService
 	Log            *zap.SugaredLogger
 	EventRecorder  *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -47,6 +52,12 @@ func (t *ProjectReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the project, ProjectReconciler implements it
 func (t *ProjectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if t.LeaderElector != nil && !t.LeaderElector.IsLeader() {
+		t.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, t.Client, t.Scheme, "ProjectController", t.EventRecorder)
 	return t.ProjectService.ReconcileProject(kubeSliceCtx, req)
 }

controllers/controller/serviceexportconfig_controller.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -36,10 +38,19 @@ type ServiceExportConfigReconciler struct {
 	ServiceExportConfigService service.IServiceExportConfigService
 	Log                        *zap.SugaredLogger
 	EventRecorder              *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // Reconcile is a function to reconcile the ServiceExportConfig, ServiceExportConfigReconciler implements it
 func (r *ServiceExportConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "ServiceExportConfigController", r.EventRecorder)
 	return r.ServiceExportConfigService.ReconcileServiceExportConfig(kubeSliceCtx, req)
 }

controllers/controller/sliceconfig_controller.go

@@ -23,6 +23,7 @@ import (
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 
@@ -38,10 +39,19 @@ type SliceConfigReconciler struct {
 	SliceConfigService service.ISliceConfigService
 	Log                *zap.SugaredLogger
 	EventRecorder      *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // Reconcile is a function to reconcile the slice config, SliceConfigReconciler implements it
 func (r *SliceConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "SliceConfigController", r.EventRecorder)
 	return r.SliceConfigService.ReconcileSliceConfig(kubeSliceCtx, req)
 }

controllers/controller/sliceqosconfig_controller.go

@@ -18,6 +18,8 @@ package controller
 
 import (
 	"context"
+
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
@@ -37,6 +39,9 @@ type SliceQoSConfigReconciler struct {
 	SliceQoSConfigService service.ISliceQoSConfigService
 	Log                   *zap.SugaredLogger
 	EventRecorder         *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -48,6 +53,12 @@ func (r *SliceQoSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the qos_profile, SliceQoSConfigReconciler implements it
 func (r *SliceQoSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "SliceQoSConfigController", r.EventRecorder)
 	return r.SliceQoSConfigService.ReconcileSliceQoSConfig(kubeSliceCtx, req)
 }