feat(ha): observability metrics, events and operator runbook (#298)

Dockerfile

@@ -23,6 +23,7 @@ COPY util/ util/
 COPY events/ events/
 COPY metrics/ metrics/
 COPY cleanup/ cleanup/
+COPY pkg/ pkg/
 
 # Build
 RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -o manager main.go

apis/controller/v1alpha1/cluster_types.go

@@ -139,6 +139,30 @@ type ClusterStatus struct {
 	// StorageCapabilities contains auto-detected storage capabilities reported by the worker operator.
 	// Populated only when the worker operator's storage-capability reconciler is active.
 	StorageCapabilities *StorageCapabilities `json:"storageCapabilities,omitempty"`
+	// ActiveController identifies the hub controller that currently holds leadership.
+	// Populated only on an Active/Standby HA deployment; absent otherwise, so a
+	// non-HA worker sees no behaviour change.
+	ActiveController *ActiveControllerInfo `json:"activeController,omitempty"`
+}
+
+// ActiveControllerInfo describes the hub controller currently holding leadership.
+//
+// Each hub writes this field about itself, on its own API server, and only while
+// it holds leadership. A Standby's copy is populated by the state mirror from the
+// Active, so it names the Active rather than itself — which lets a worker watching
+// both hubs identify the Active by the rule "trust whichever endpoint is reachable
+// and reports an ActiveIdentity matching that endpoint's own identity", without
+// needing to know which role either hub currently holds. See ADR #293 Decision 7.
+type ActiveControllerInfo struct {
+	// Endpoint is the API server endpoint of the hub currently holding leadership
+	Endpoint string `json:"endpoint,omitempty"`
+	// CABundle is the base64-encoded PEM CA bundle for Endpoint
+	CABundle string `json:"caBundle,omitempty"`
+	// ActiveIdentity is the HA identity of the hub that wrote this field about itself
+	ActiveIdentity string `json:"activeIdentity,omitempty"`
+	// LastUpdated is the timestamp when this declaration was last written. It gives
+	// a consumer a deterministic tie-break if both hubs self-declare simultaneously.
+	LastUpdated metav1.Time `json:"lastUpdated,omitempty"`
 }
 
 // StorageCapabilities holds auto-detected RWX-capable storage classes on the worker cluster.

config/crd/bases/controller.kubeslice.io_clusters.yaml

@@ -133,6 +133,31 @@ spec:
                     format: date-time
                     type: string
                 type: object
+              activeController:
+                description: |-
+                  ActiveController identifies the hub controller that currently holds leadership.
+                  Populated only on an Active/Standby HA deployment; absent otherwise, so a
+                  non-HA worker sees no behaviour change.
+                properties:
+                  activeIdentity:
+                    description: ActiveIdentity is the HA identity of the hub that
+                      wrote this field about itself
+                    type: string
+                  caBundle:
+                    description: CABundle is the base64-encoded PEM CA bundle for
+                      Endpoint
+                    type: string
+                  endpoint:
+                    description: Endpoint is the API server endpoint of the hub currently
+                      holding leadership
+                    type: string
+                  lastUpdated:
+                    description: |-
+                      LastUpdated is the timestamp when this declaration was last written. It gives
+                      a consumer a deterministic tie-break if both hubs self-declare simultaneously.
+                    format: date-time
+                    type: string
+                type: object
               clusterHealth:
                 description: ClusterHealth shows the health of the worker cluster
                 properties:

config/default/manager_auth_proxy_patch.yaml

@@ -24,6 +24,19 @@ spec:
           args:
             - "--health-probe-bind-address=:8081"
             - "--metrics-bind-address=127.0.0.1:8080"
+            # Required for the kube-rbac-proxy sidecar above to be able to scrape
+            # this manager at all. --metrics-secure defaults to TRUE, which makes
+            # controller-runtime serve the metrics endpoint over TLS *and* wrap it
+            # in its own authn/authz filter — while the sidecar is configured with
+            # "--upstream=http://127.0.0.1:8080/", plain HTTP. Left at the default
+            # the proxy speaks HTTP to a TLS listener and /metrics is unreachable
+            # through the only path the Service exposes.
+            #
+            # Plain HTTP on loopback is the intended kube-rbac-proxy arrangement,
+            # not a weakening of it: the endpoint is bound to 127.0.0.1 so nothing
+            # outside the pod can reach it directly, and the sidecar is what
+            # terminates TLS on 8443 and authorizes callers by SubjectAccessReview.
+            - "--metrics-secure=false"
             - "--leader-elect"
             - "--log-level=debug"
             - "--rbac-resource-prefix=kubeslice-rbac"

config/events/controller.yaml

@@ -539,3 +539,39 @@ events:
     type: Warning
     reportingController: controller
     message: Warning - Certificate Creation job Failed
+  - name: HAMirrorSyncFailed
+    reason: HAMirrorSyncFailed
+    action: HAMirrorSync
+    type: Warning
+    reportingController: controller
+    message: Failed to mirror a resource from the Active hub onto the Standby; the syncer will retry.
+  - name: HAPromotedToActive
+    reason: PromotedToActive
+    action: HAPromotion
+    type: Normal
+    reportingController: controller
+    message: This hub was promoted from Standby to Active after the previous Active hub's lease went stale.
+  - name: HABecameActive
+    reason: BecameActive
+    action: HAStartup
+    type: Normal
+    reportingController: controller
+    message: This hub started in Active mode and will hold the HA lease and reconcile.
+  - name: HABecameStandby
+    reason: BecameStandby
+    action: HAStartup
+    type: Normal
+    reportingController: controller
+    message: This hub started in Standby mode; it mirrors the Active hub's state and does not reconcile.
+  - name: HALeadershipLost
+    reason: LeadershipLost
+    action: HALeaseRenewal
+    type: Warning
+    reportingController: controller
+    message: This hub failed to renew its HA lease within the renew deadline and has released leadership; it will not reconcile until it renews again.
+  - name: HAPromotionAborted
+    reason: PromotionAborted
+    action: HAPromotion
+    type: Warning
+    reportingController: controller
+    message: A promotion was considered and refused; see ha_promotions_aborted_total and the controller logs for which guard fired.

config/events/events_config_map.yaml

@@ -96,4 +96,10 @@ data:
       - CertificateJobCreationFailed
       - CertificatesRenewNow
       - IllegalVPNKeyRotationConfigDelete
-      - CertificateJobFailed
+      - CertificateJobFailed
+      - HAMirrorSyncFailed
+      - HAPromotedToActive
+      - HABecameActive
+      - HABecameStandby
+      - HALeadershipLost
+      - HAPromotionAborted

config/ha/README.md

@@ -0,0 +1,59 @@
+# HA cross-cluster RBAC (issue #295)
+
+`active-cluster-clusterrole.yaml` is a least-privilege grant for the
+identity behind a Standby's `--ha-active-kubeconfig` flag: read-only
+(`get`/`list`/`watch`) access to `Namespace` plus every resource type in
+`pkg/ha.CRDMirrorSet` and `pkg/ha.CredentialMirrorSet` (`RemoteSyncer`
+never writes to the Active cluster — only reads), plus the Active's own
+`coordination.k8s.io/v1` `Lease` — the same kubeconfig is also used by
+#294's `WatchRemoteLease` to read the Active's Lease directly, not just by
+`RemoteSyncer` to mirror resources.
+
+## This is not applied by this repo's own deploy flow
+
+Nothing here is referenced by `config/rbac/kustomization.yaml` or
+`config/default/kustomization.yaml`. Those govern the RBAC a controller
+grants *itself* on the cluster it's running in. This manifest is different:
+it must be applied **on the Active hub cluster**, granting access to
+whatever identity the *Standby's* kubeconfig authenticates as — a cluster
+this repo's own kustomize overlays have no way to reach, since it's a
+separate cluster entirely.
+
+Apply it manually (or via whatever provisioning tooling manages the Active
+hub) against the Active cluster:
+
+```
+kubectl --context <active-hub-context> apply -f active-cluster-clusterrole.yaml
+```
+
+Fill in the `ClusterRoleBinding`'s `subjects` first — the correct subject
+depends on how the Standby authenticates to the Active (a `ServiceAccount`
+if dialing in-cluster, a client-cert `User` if using a flattened
+kubeconfig Secret, as the current dev demo does).
+
+## A known, deliberate gap this manifest does not close
+
+Nothing in this repo automates applying this to a real Active cluster —
+that's cross-cluster provisioning, out of scope for a single controller
+repo. Logged as a follow-up, not built.
+
+## Credential mirroring and the Secret-read tradeoff
+
+`pkg/ha.CredentialMirrorSet` (Secrets with the SA-token type filtered out,
+ServiceAccounts, Roles, RoleBindings — for #297's post-promotion use) is
+wired in, and this `ClusterRole` grants the reads it needs. Weigh the
+Secret rule before applying it: RBAC cannot scope `Secret` access by
+`.type` or by namespace *label*, and a `ClusterRole` +
+`ClusterRoleBinding` is cluster-wide — so the Standby's identity can read
+**every** Secret on the Active hub, not just the gateway-certificate
+Secrets `RemoteSyncer` actually mirrors. The syncer itself only *copies*
+credential objects whose namespace it also mirrors (the label-scoped
+project-namespace boundary — notably excluding the controller's own
+namespace, whose name can match the project-namespace prefix) and
+excludes SA-token Secrets from the watch entirely, but none of that
+narrows what the identity *could* read if the kubeconfig leaked —
+protect it like the credential it is. The narrower alternative — per-namespace `RoleBinding`s in each
+project namespace instead of the cluster-wide binding — works with the
+same `ClusterRole`, at the cost of maintaining those bindings as projects
+come and go (cross-cluster provisioning tooling this repo deliberately
+does not ship).

config/ha/active-cluster-clusterrole.yaml

@@ -0,0 +1,113 @@
+---
+# Least-privilege grant for the identity behind the Standby's
+# --ha-active-kubeconfig, applied ON THE ACTIVE CLUSTER — not part of this
+# repo's own config/rbac (that governs the local cluster's own ClusterRole)
+# and not referenced by config/rbac/kustomization.yaml or
+# config/default/kustomization.yaml, so it is never auto-applied to the
+# Standby's own cluster, where it would be meaningless.
+#
+# See config/ha/README.md for how and where to apply this.
+#
+# Scope: read-only (get/list/watch) on Namespace plus every type in
+# pkg/ha.CRDMirrorSet and pkg/ha.CredentialMirrorSet, plus the Active's own
+# HA Lease — the same --ha-active-kubeconfig identity is also used by #294's
+# WatchRemoteLease to read the Active's coordination.k8s.io/v1 Lease
+# (checkRemoteLeaseOnce), not just by RemoteSyncer. Confirmed live: without
+# this, the Standby can mirror CRDs fine but permanently fails to read the
+# Active's Lease, so it can never observe staleness in the first place.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubeslice-ha-standby-reader
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - controller.kubeslice.io
+    resources:
+      - projects
+      - clusters
+      - sliceconfigs
+      - serviceexportconfigs
+      - sliceqosconfigs
+      - vpnkeyrotations
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - worker.kubeslice.io
+    resources:
+      - workersliceconfigs
+      - workerslicegateways
+      - workerserviceimports
+    verbs:
+      - get
+      - list
+      - watch
+  # Credential mirroring (pkg/ha.CredentialMirrorSet). SECURITY TRADEOFF,
+  # named rather than buried: RBAC cannot scope Secret access by .type or by
+  # namespace *label*, and a ClusterRole+ClusterRoleBinding grants reads
+  # CLUSTER-WIDE — so this rule lets the Standby's identity read every
+  # Secret on the Active hub, not just the ones RemoteSyncer actually
+  # mirrors. The syncer's mirrored-namespace gate narrows what is COPIED,
+  # never what this identity COULD read; treat the --ha-active-kubeconfig
+  # credential accordingly. To narrow the grant itself, replace this rule
+  # with per-namespace RoleBindings in each kubeslice project namespace — at
+  # the cost of maintaining them as projects come and go, which is
+  # cross-cluster provisioning tooling this repo deliberately does not ship.
+  #
+  # Service-account-token Secrets are within this grant and are read
+  # deliberately: the Standby mirrors their shells so its own token
+  # controller can mint a worker credential valid on itself before any
+  # failover. The token VALUES never leave the Active — they are dropped on
+  # the way into the Standby's cache and again before any write.
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+      - serviceaccounts
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - roles
+      - rolebindings
+    verbs:
+      - get
+      - list
+      - watch
+---
+# Template only — the subject is deployment-specific (a ServiceAccount if the
+# Standby dials the Active in-cluster, a cert CN if it uses client-cert auth
+# via a flattened kubeconfig) and can't be hardcoded here. Fill in `subjects`
+# before applying.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kubeslice-ha-standby-reader-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubeslice-ha-standby-reader
+subjects:
+  - kind: User # or ServiceAccount — see README.md
+    name: CHANGEME
+    apiGroup: rbac.authorization.k8s.io

config/rbac/role.yaml

@@ -30,6 +30,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - ""
   resources: