feat(ha): cross-cluster leader election + strict write fencing (Active-only reconcile)

controllers/controller/cluster_controller.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -36,6 +38,9 @@ type ClusterReconciler struct {
 	ClusterService service.IClusterService
 	Log            *zap.SugaredLogger
 	EventRecorder  *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -47,6 +52,12 @@ func (c *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the cluster , ClusterReconciler implements it
 func (c *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if c.LeaderElector != nil && !c.LeaderElector.IsLeader() {
+		c.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, c.Client, c.Scheme, "ClusterController", c.EventRecorder)
 	return c.ClusterService.ReconcileCluster(kubeSliceCtx, req)
 }

controllers/controller/leader_gate_test.go

@@ -0,0 +1,62 @@
+/*
+ * 	Copyright (c) 2022 Avesha, Inc. All rights reserved. # # SPDX-License-Identifier: Apache-2.0
+ *
+ * 	Licensed under the Apache License, Version 2.0 (the "License");
+ * 	you may not use this file except in compliance with the License.
+ * 	You may obtain a copy of the License at
+ *
+ * 	http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * 	Unless required by applicable law or agreed to in writing, software
+ * 	distributed under the License is distributed on an "AS IS" BASIS,
+ * 	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * 	See the License for the specific language governing permissions and
+ * 	limitations under the License.
+ */
+
+package controller
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+	"go.uber.org/zap/zaptest/observer"
+	ctrl "sigs.k8s.io/controller-runtime"
+
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
+)
+
+// TestReconcile_StandbySkipsAndLogs verifies the HA write fence: a Standby
+// controller returns immediately from Reconcile without touching its service
+// (left nil here — a leaking gate would panic), and logs the skip message on
+// every call, proving IsLeader() is evaluated per invocation.
+func TestReconcile_StandbySkipsAndLogs(t *testing.T) {
+	core, logs := observer.New(zapcore.InfoLevel)
+	logger := zap.New(core).Sugar()
+
+	standby := ha.NewClusterLeaderElector(nil, nil, ha.Options{
+		Mode: ha.ModeStandby,
+		Log:  zap.NewNop().Sugar(),
+	})
+	require.False(t, standby.IsLeader(), "standby must not be leader")
+
+	r := &SliceConfigReconciler{
+		Log:           logger,
+		LeaderElector: standby,
+		// SliceConfigService is intentionally nil: the gate must return before it.
+	}
+
+	const calls = 3
+	for i := 0; i < calls; i++ {
+		res, err := r.Reconcile(context.Background(), ctrl.Request{})
+		require.NoError(t, err)
+		assert.Equal(t, ctrl.Result{}, res)
+	}
+
+	assert.Equal(t, calls, logs.FilterMessage("standby mode, skipping reconcile").Len(),
+		"expected one skip log per Reconcile call")
+}

controllers/controller/project_controller.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -36,6 +38,9 @@ type ProjectReconciler struct {
 	ProjectService service.IProjectService
 	Log            *zap.SugaredLogger
 	EventRecorder  *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -47,6 +52,12 @@ func (t *ProjectReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the project, ProjectReconciler implements it
 func (t *ProjectReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if t.LeaderElector != nil && !t.LeaderElector.IsLeader() {
+		t.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, t.Client, t.Scheme, "ProjectController", t.EventRecorder)
 	return t.ProjectService.ReconcileProject(kubeSliceCtx, req)
 }

controllers/controller/serviceexportconfig_controller.go

@@ -18,10 +18,12 @@ package controller
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -36,10 +38,19 @@ type ServiceExportConfigReconciler struct {
 	ServiceExportConfigService service.IServiceExportConfigService
 	Log                        *zap.SugaredLogger
 	EventRecorder              *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // Reconcile is a function to reconcile the ServiceExportConfig, ServiceExportConfigReconciler implements it
 func (r *ServiceExportConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "ServiceExportConfigController", r.EventRecorder)
 	return r.ServiceExportConfigService.ReconcileServiceExportConfig(kubeSliceCtx, req)
 }

controllers/controller/sliceconfig_controller.go

@@ -23,6 +23,7 @@ import (
 	"go.uber.org/zap"
 
 	controllerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/controller/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 
@@ -38,10 +39,19 @@ type SliceConfigReconciler struct {
 	SliceConfigService service.ISliceConfigService
 	Log                *zap.SugaredLogger
 	EventRecorder      *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // Reconcile is a function to reconcile the slice config, SliceConfigReconciler implements it
 func (r *SliceConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "SliceConfigController", r.EventRecorder)
 	return r.SliceConfigService.ReconcileSliceConfig(kubeSliceCtx, req)
 }

controllers/controller/sliceqosconfig_controller.go

@@ -18,6 +18,8 @@ package controller
 
 import (
 	"context"
+
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
@@ -37,6 +39,9 @@ type SliceQoSConfigReconciler struct {
 	SliceQoSConfigService service.ISliceQoSConfigService
 	Log                   *zap.SugaredLogger
 	EventRecorder         *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -48,6 +53,12 @@ func (r *SliceQoSConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the qos_profile, SliceQoSConfigReconciler implements it
 func (r *SliceQoSConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "SliceQoSConfigController", r.EventRecorder)
 	return r.SliceQoSConfigService.ReconcileSliceQoSConfig(kubeSliceCtx, req)
 }

controllers/controller/vpnkey_rotation_controller.go

@@ -18,6 +18,8 @@ package controller
 
 import (
 	"context"
+
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
@@ -37,6 +39,9 @@ type VpnKeyRotationReconciler struct {
 	VpnKeyRotationService service.IVpnKeyRotationService
 	Log                   *zap.SugaredLogger
 	EventRecorder         *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -48,6 +53,12 @@ func (r *VpnKeyRotationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcile the VpnKeyRotation, VpnKeyRotationReconciler implements it
 func (r *VpnKeyRotationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "VpnKeyRotationController", r.EventRecorder)
 	return r.VpnKeyRotationService.ReconcileVpnKeyRotation(kubeSliceCtx, req)
 }

controllers/worker/workerserviceimport_controller.go

@@ -18,10 +18,12 @@ package worker
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	"github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -37,10 +39,19 @@ type WorkerServiceImportReconciler struct {
 	WorkerServiceImportService service.IWorkerServiceImportService
 	Log                        *zap.SugaredLogger
 	EventRecorder              *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // Reconcile is a function to reconcile the workerServiceImport, WorkerServiceImportReconciler implements it
 func (r *WorkerServiceImportReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "WorkerServiceImportController", r.EventRecorder)
 	return r.WorkerServiceImportService.ReconcileWorkerServiceImport(kubeSliceCtx, req)
 }

controllers/worker/workersliceconfig_controller.go

@@ -18,14 +18,15 @@ package worker
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
+	workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 
-	workerv1alpha1 "github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1"
-
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -38,6 +39,9 @@ type WorkerSliceConfigReconciler struct {
 	WorkerSliceService service.IWorkerSliceConfigService
 	Log                *zap.SugaredLogger
 	EventRecorder      *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -49,6 +53,12 @@ func (c *WorkerSliceConfigReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 // Reconcile is a function to reconcilation of WorkerSliceconfig, WorkerSliceConfigReconciler implements it
 func (c *WorkerSliceConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if c.LeaderElector != nil && !c.LeaderElector.IsLeader() {
+		c.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, c.Client, c.Scheme, "WorkerSliceConfigController", c.EventRecorder)
 	return c.WorkerSliceService.ReconcileWorkerSliceConfig(kubeSliceCtx, req)
 }

controllers/worker/workerslicegateway_controller.go

@@ -18,10 +18,12 @@ package worker
 
 import (
 	"context"
+
 	"github.com/kubeslice/kubeslice-monitoring/pkg/events"
 	"go.uber.org/zap"
 
 	"github.com/kubeslice/kubeslice-controller/apis/worker/v1alpha1"
+	"github.com/kubeslice/kubeslice-controller/pkg/ha"
 	"github.com/kubeslice/kubeslice-controller/service"
 	"github.com/kubeslice/kubeslice-controller/util"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -37,10 +39,19 @@ type WorkerSliceGatewayReconciler struct {
 	WorkerSliceGatewayService service.IWorkerSliceGatewayService
 	Log                       *zap.SugaredLogger
 	EventRecorder             *events.EventRecorder
+	// LeaderElector gates mutating reconciles on cross-cluster leadership. It is
+	// nil-safe: a nil elector (HA not wired) behaves as standalone. See ADR #293.
+	LeaderElector *ha.ClusterLeaderElector
 }
 
 // Reconcile is a function, WorkerSliceGatewayReconciler implements it
 func (r *WorkerSliceGatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	// HA write fence: only the Active hub (or a standalone controller) writes.
+	// A Standby evaluates this on every call and no-ops.
+	if r.LeaderElector != nil && !r.LeaderElector.IsLeader() {
+		r.Log.Info("standby mode, skipping reconcile")
+		return ctrl.Result{}, nil
+	}
 	kubeSliceCtx := util.PrepareKubeSliceControllersRequestContext(ctx, r.Client, r.Scheme, "WorkerSliceGatewayController", r.EventRecorder)
 	return r.WorkerSliceGatewayService.ReconcileWorkerSliceGateways(kubeSliceCtx, req)
 }