Add splitbrain condition, make tests terser

Author: lllamnyp

api/v1alpha1/etcdcluster_types.go

@@ -55,6 +55,7 @@ type EtcdClusterSpec struct {
 const (
 	EtcdConditionInitialized = "Initialized"
 	EtcdConditionReady       = "Ready"
+	EtcdConditionError       = "Error"
 )
 
 type EtcdCondType string
@@ -66,6 +67,7 @@ const (
 	EtcdCondTypeWaitingForFirstQuorum EtcdCondType = "WaitingForFirstQuorum"
 	EtcdCondTypeStatefulSetReady      EtcdCondType = "StatefulSetReady"
 	EtcdCondTypeStatefulSetNotReady   EtcdCondType = "StatefulSetNotReady"
+	EtcdCondTypeSplitbrain            EtcdCondType = "Splitbrain"
 )
 
 const (
@@ -74,6 +76,7 @@ const (
 	EtcdReadyCondNegMessage          EtcdCondMessage = "Cluster StatefulSet is not Ready"
 	EtcdReadyCondPosMessage          EtcdCondMessage = "Cluster StatefulSet is Ready"
 	EtcdReadyCondNegWaitingForQuorum EtcdCondMessage = "Waiting for first quorum to be established"
+	EtcdErrorCondSplitbrainMessage   EtcdCondMessage = "Etcd endpoints reporting more than one unique cluster ID"
 )
 
 // EtcdClusterStatus defines the observed state of EtcdCluster

internal/controller/etcdcluster_controller.go

@@ -133,7 +133,13 @@ func (r *EtcdClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	state.setClusterID()
 	if state.inSplitbrain() {
 		log.Error(ctx, fmt.Errorf("etcd cluster in splitbrain"), "etcd cluster in splitbrain, dropping from reconciliation queue")
-		return ctrl.Result{}, nil
+		factory.SetCondition(instance, factory.NewCondition(etcdaenixiov1alpha1.EtcdConditionError).
+			WithStatus(true).
+			WithReason(string(etcdaenixiov1alpha1.EtcdCondTypeSplitbrain)).
+			WithMessage(string(etcdaenixiov1alpha1.EtcdErrorCondSplitbrainMessage)).
+			Complete(),
+		)
+		return r.updateStatus(ctx, instance)
 	}
 	// fill conditions
 	if len(instance.Status.Conditions) == 0 {

test/e2e/e2e_test.go

@@ -97,24 +97,15 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			})
 
 			Eventually(func() error {
-				cmd := exec.Command("kubectl", "get",
-					"statefulset/test",
-					"--namespace", namespace,
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed())
-
-			By("wait for statefulset is ready", func() {
 				cmd := exec.Command("kubectl", "wait",
 					"statefulset/test",
 					"--for", "jsonpath={.status.readyReplicas}=3",
 					"--namespace", namespace,
 					"--timeout", "5m",
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			})
+				return err
+			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
 			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
 			Expect(err).NotTo(HaveOccurred())
@@ -155,24 +146,15 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			})
 
 			Eventually(func() error {
-				cmd := exec.Command("kubectl", "get",
-					"statefulset/test",
-					"--namespace", namespace,
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed())
-
-			By("wait for statefulset is ready", func() {
 				cmd := exec.Command("kubectl", "wait",
 					"statefulset/test",
 					"--for", "jsonpath={.status.readyReplicas}=3",
 					"--namespace", namespace,
 					"--timeout", "5m",
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			})
+				return err
+			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
 			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
 			Expect(err).NotTo(HaveOccurred())
@@ -212,24 +194,15 @@ var _ = Describe("etcd-operator", Ordered, func() {
 			})
 
 			Eventually(func() error {
-				cmd := exec.Command("kubectl", "get",
-					"statefulset/test",
-					"--namespace", namespace,
-				)
-				_, err = utils.Run(cmd)
-				return err
-			}, time.Second*20, time.Second*2).Should(Succeed())
-
-			By("wait for statefulset is ready", func() {
 				cmd := exec.Command("kubectl", "wait",
 					"statefulset/test",
 					"--for", "jsonpath={.status.availableReplicas}=3",
 					"--namespace", namespace,
 					"--timeout", "5m",
 				)
 				_, err = utils.Run(cmd)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred())
-			})
+				return err
+			}, time.Second*20, time.Second*2).Should(Succeed(), "wait for statefulset is ready")
 
 			client, err := utils.GetEtcdClient(ctx, client.ObjectKey{Namespace: namespace, Name: "test"})
 			Expect(err).NotTo(HaveOccurred())