ocpbugs-57524: improve stability of MCN slow tests

Author: isabella-janssen

test/extended/machine_config/helpers.go

@@ -735,17 +735,28 @@ func CheckMCNConditionStatus(mcn *mcfgv1.MachineConfigNode, conditionType mcfgv1
 	return conditionStatus == status
 }
 
-// `getMCNConditionStatus` returns the status of the desired condition type for MCN, or an empty string if the condition does not exist
-func getMCNConditionStatus(mcn *mcfgv1.MachineConfigNode, conditionType mcfgv1.StateProgress) metav1.ConditionStatus {
+// `GetMCNCondition` returns the queried condition or nil if the condition does not exist
+func GetMCNCondition(mcn *mcfgv1.MachineConfigNode, conditionType mcfgv1.StateProgress) *metav1.Condition {
 	// Loop through conditions and return the status of the desired condition type
 	conditions := mcn.Status.Conditions
 	for _, condition := range conditions {
 		if condition.Type == string(conditionType) {
-			framework.Logf("MCN '%s' %s condition status is %s", mcn.Name, conditionType, condition.Status)
-			return condition.Status
+			return &condition
 		}
 	}
-	return ""
+	return nil
+}
+
+// `getMCNConditionStatus` returns the status of the desired condition type for MCN, or an empty string if the condition does not exist
+func getMCNConditionStatus(mcn *mcfgv1.MachineConfigNode, conditionType mcfgv1.StateProgress) metav1.ConditionStatus {
+	// Loop through conditions and return the status of the desired condition type
+	condition := GetMCNCondition(mcn, conditionType)
+	if condition == nil {
+		return ""
+	}
+
+	framework.Logf("MCN '%s' %s condition status is %s", mcn.Name, conditionType, condition.Status)
+	return condition.Status
 }
 
 // `ConfirmUpdatedMCNStatus` confirms that an MCN is in a fully updated state, which requires:
@@ -994,7 +1005,7 @@ func GetNodeInMachine(oc *exutil.CLI, machineName string) (corev1.Node, error) {
 	return *node, nil
 }
 
-// `GetNewReadyNodeInMachine` waits up to 2 minutes for the newly provisioned node in a desired machine node to be ready
+// `GetNewReadyNodeInMachine` waits up to 4 minutes for the newly provisioned node in a desired machine node to be ready
 func GetNewReadyNodeInMachine(oc *exutil.CLI, machineName string) (corev1.Node, error) {
 	desiredNode := corev1.Node{}
 	err := fmt.Errorf("no ready node in Machine: %s", machineName)
@@ -1013,7 +1024,7 @@ func GetNewReadyNodeInMachine(oc *exutil.CLI, machineName string) (corev1.Node,
 		}
 
 		return false
-	}, 2*time.Minute, 3*time.Second).Should(o.BeTrue())
+	}, 4*time.Minute, 5*time.Second).Should(o.BeTrue(), fmt.Sprintf("Node in machine %v never became ready.", machineName))
 	return desiredNode, err
 }
 

test/extended/machine_config/machine_config_node.go

@@ -371,6 +371,7 @@ func ValidateMCNConditionOnNodeDegrade(oc *exutil.CLI, fixture string, isSno boo
 		mcName = "91-master-testfile-invalid"
 	}
 
+	var degradedNodeMCN *mcfgv1.MachineConfigNode
 	// Cleanup MC and fix node degradation on failure or test completion
 	defer func() {
 		// Delete the applied MC
@@ -380,6 +381,13 @@ func ValidateMCNConditionOnNodeDegrade(oc *exutil.CLI, fixture string, isSno boo
 		// Recover the degraded MCP
 		recoverErr := RecoverFromDegraded(oc, poolName)
 		o.Expect(recoverErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not recover MCP '%v' from degraded state.", poolName))
+
+		// If the test reached checking the MCN ensure the NodeDegraded condition is properly restored
+		if degradedNodeMCN != nil {
+			conditionMet, err := WaitForMCNConditionStatus(clientSet, degradedNodeMCN.Name, mcfgv1.MachineConfigNodeNodeDegraded, metav1.ConditionFalse, 30*time.Second, 1*time.Second)
+			o.Expect(err).NotTo(o.HaveOccurred(), fmt.Sprintf("Error occured while waiting for NodeDegraded=False: %v", err))
+			o.Expect(conditionMet).To(o.BeTrue(), "Error, could not detect NodeDegraded=False.")
+		}
 	}()
 
 	// Apply invalid MC
@@ -398,11 +406,27 @@ func ValidateMCNConditionOnNodeDegrade(oc *exutil.CLI, fixture string, isSno boo
 	o.Expect(degradedNodeErr).NotTo(o.HaveOccurred(), "Could not get degraded node.")
 
 	// Validate MCN of degraded node
-	degradedNodeMCN, degradedErr := clientSet.MachineconfigurationV1().MachineConfigNodes().Get(context.TODO(), degradedNode.Name, metav1.GetOptions{})
+	// 	get and log MCN conditions for debugging purposes
+	degradedNodeMCN, degradedErr = clientSet.MachineconfigurationV1().MachineConfigNodes().Get(context.TODO(), degradedNode.Name, metav1.GetOptions{})
 	o.Expect(degradedErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error getting MCN of degraded node '%v'.", degradedNode.Name))
-	framework.Logf("Validating that `AppliedFilesAndOS` and `UpdateExecuted` conditions in '%v' MCN have a status of 'Unknown'.", degradedNodeMCN.Name)
-	o.Expect(CheckMCNConditionStatus(degradedNodeMCN, mcfgv1.MachineConfigNodeUpdateFilesAndOS, metav1.ConditionUnknown)).Should(o.BeTrue(), "Condition 'AppliedFilesAndOS' does not have the expected status of 'Unknown'.")
-	o.Expect(CheckMCNConditionStatus(degradedNodeMCN, mcfgv1.MachineConfigNodeUpdateExecuted, metav1.ConditionUnknown)).Should(o.BeTrue(), "Condition 'UpdateExecuted' does not have the expected status of 'Unknown'.")
+	nodeDegradedCondition := GetMCNCondition(degradedNodeMCN, mcfgv1.MachineConfigNodeNodeDegraded)
+	o.Expect(nodeDegradedCondition).NotTo(o.BeNil(), "Condition 'NodeDegraded' does not exist.")
+	framework.Logf("`NodeDegraded` condition status is `%v` with the message `%v`", nodeDegradedCondition.Status, nodeDegradedCondition.Message)
+	filesAndOSCondition := GetMCNCondition(degradedNodeMCN, mcfgv1.MachineConfigNodeUpdateFilesAndOS)
+	o.Expect(filesAndOSCondition).NotTo(o.BeNil(), "Condition 'AppliedFilesAndOS' does not exist.")
+	framework.Logf("`AppliedFilesAndOS` condition status is `%v` with the message `%v`", filesAndOSCondition.Status, filesAndOSCondition.Message)
+	executedCondition := GetMCNCondition(degradedNodeMCN, mcfgv1.MachineConfigNodeUpdateExecuted)
+	o.Expect(executedCondition).NotTo(o.BeNil(), "Condition 'UpdateExecuted' does not exist.")
+	framework.Logf("`UpdateExecuted` condition status is `%v` with the message `%v`", executedCondition.Status, executedCondition.Message)
+	// 	validate the conditions are as expected
+	framework.Logf("Validating that `NodeDegraded` condition in '%v' MCN has a status of 'True'.", degradedNodeMCN.Name)
+	o.Expect(nodeDegradedCondition.Status).Should(o.Equal(metav1.ConditionTrue), "Condition 'NodeDegraded' does not have the expected status of 'True'.")
+	o.Expect(nodeDegradedCondition.Message).Should(o.ContainSubstring(fmt.Sprintf("Node %s upgrade failure.", degradedNodeMCN.Name)), "Condition 'NodeDegraded' does not have the expected message.")
+	o.Expect(nodeDegradedCondition.Message).Should(o.ContainSubstring("/home/core: file exists"), "Condition 'NodeDegraded' does not have the expected message details.")
+	framework.Logf("Validating that `UpdateExecuted` condition in '%v' MCN has a status of 'Unknown'.", degradedNodeMCN.Name)
+	o.Expect(executedCondition.Status).Should(o.Equal(metav1.ConditionUnknown), "Condition 'UpdateExecuted' does not have the expected status of 'Unknown'.")
+	framework.Logf("Validating that `AppliedFilesAndOS` condition in '%v' MCN has a status of 'Unknown'.", degradedNodeMCN.Name)
+	o.Expect(filesAndOSCondition.Status).Should(o.Equal(metav1.ConditionUnknown), "Condition 'AppliedFilesAndOS' does not have the expected status of 'Unknown'.")
 }
 
 // `ValidateMCNProperties` checks that MCNs with correct properties are created on node creation