diff --git a/pkg/controllers/updaterun/controller.go b/pkg/controllers/updaterun/controller.go index 9a0ee4862..2a8a742c4 100644 --- a/pkg/controllers/updaterun/controller.go +++ b/pkg/controllers/updaterun/controller.go @@ -104,11 +104,26 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim // Emit the update run status metric based on status conditions in the updateRun. defer emitUpdateRunStatusMetric(updateRun) + state := updateRun.GetUpdateRunSpec().State + switch state { // Early check for abandoned state - this is a terminal state, no initialization needed. + case placementv1beta1.StateAbandoned: + klog.V(2).InfoS("The updateRun is abandoned, terminating", "state", state, "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunAbandoned(ctx, updateRun) + case placementv1beta1.StateStopped: // Early check for stopped state - pause the update run if needed. + klog.V(2).InfoS("The updateRun is paused, waiting to resume", "state", state, "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunPaused(ctx, updateRun) + } + var updatingStageIndex int var toBeUpdatedBindings, toBeDeletedBindings []placementv1beta1.BindingObj updateRunStatus := updateRun.GetUpdateRunStatus() initCond := meta.FindStatusCondition(updateRunStatus.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) - if !condition.IsConditionStatusTrue(initCond, updateRun.GetGeneration()) { + // Check if initialized regardless of generation. + // The updateRun spec fields are immutable except for the state field. When the state changes, + // the update run generation increments, but we don't need to reinitialize since initialization is a one-time setup. + isInitialized := initCond != nil && initCond.Status == metav1.ConditionTrue + if !isInitialized { + // Check if initialization failed for the current generation. if condition.IsConditionStatusFalse(initCond, updateRun.GetGeneration()) { klog.V(2).InfoS("The updateRun has failed to initialize", "errorMsg", initCond.Message, "updateRun", runObjRef) return runtime.Result{}, nil @@ -122,7 +137,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } return runtime.Result{}, initErr } - updatingStageIndex = 0 // start from the first stage. + updatingStageIndex = 0 // start from the first stage (typically for NotStarted or Started states). klog.V(2).InfoS("Initialized the updateRun", "updateRun", runObjRef) } else { klog.V(2).InfoS("The updateRun is initialized", "updateRun", runObjRef) @@ -134,6 +149,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } var validateErr error // Validate the updateRun status to ensure the update can be continued and get the updating stage index and cluster indices. + // For Stopped → Started transition, this will resume from where it left off. if updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings, validateErr = r.validate(ctx, updateRun); validateErr != nil { // errStagedUpdatedAborted cannot be retried. if errors.Is(validateErr, errStagedUpdatedAborted) { @@ -151,28 +167,32 @@ func (r *Reconciler) Reconcile(ctx context.Context, req runtime.Request) (runtim } // Execute the updateRun. - klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) - finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) - if errors.Is(execErr, errStagedUpdatedAborted) { - // errStagedUpdatedAborted cannot be retried. - return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) - } + if state == placementv1beta1.StateStarted { + klog.V(2).InfoS("Continue to execute the updateRun", "updatingStageIndex", updatingStageIndex, "updateRun", runObjRef) + finished, waitTime, execErr := r.execute(ctx, updateRun, updatingStageIndex, toBeUpdatedBindings, toBeDeletedBindings) + if errors.Is(execErr, errStagedUpdatedAborted) { + // errStagedUpdatedAborted cannot be retried. + return runtime.Result{}, r.recordUpdateRunFailed(ctx, updateRun, execErr.Error()) + } - if finished { - klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) - return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) - } + if finished { + klog.V(2).InfoS("The updateRun is completed", "updateRun", runObjRef) + return runtime.Result{}, r.recordUpdateRunSucceeded(ctx, updateRun) + } - // The execution is not finished yet or it encounters a retriable error. - // We need to record the status and requeue. - if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { - return runtime.Result{}, updateErr - } - klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) - if execErr != nil { - return runtime.Result{}, execErr + // The execution is not finished yet or it encounters a retriable error. + // We need to record the status and requeue. + if updateErr := r.recordUpdateRunStatus(ctx, updateRun); updateErr != nil { + return runtime.Result{}, updateErr + } + klog.V(2).InfoS("The updateRun is not finished yet", "requeueWaitTime", waitTime, "execErr", execErr, "updateRun", runObjRef) + if execErr != nil { + return runtime.Result{}, execErr + } + return runtime.Result{RequeueAfter: waitTime}, nil } - return runtime.Result{Requeue: true, RequeueAfter: waitTime}, nil + klog.V(2).InfoS("The updateRun is not started, waiting to be started", "state", state, "updateRun", runObjRef) + return runtime.Result{}, nil } // handleDelete handles the deletion of the updateRun object. @@ -265,6 +285,50 @@ func (r *Reconciler) recordUpdateRunFailed(ctx context.Context, updateRun placem return nil } +// recordUpdateRunPaused records the progressing condition as paused in the updateRun status. +func (r *Reconciler) recordUpdateRunPaused(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionFalse, + ObservedGeneration: updateRun.GetGeneration(), + Reason: condition.UpdateRunPausedReason, + Message: "The update run is paused", + }) + if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { + klog.ErrorS(updateErr, "Failed to update the updateRun status as paused", "updateRun", klog.KObj(updateRun)) + // updateErr can be retried. + return controller.NewUpdateIgnoreConflictError(updateErr) + } + return nil +} + +// recordUpdateRunAbandoned records the succeeded and progressing condition as abandoned in the updateRun status. +func (r *Reconciler) recordUpdateRunAbandoned(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { + updateRunStatus := updateRun.GetUpdateRunStatus() + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionFalse, + ObservedGeneration: updateRun.GetGeneration(), + Reason: condition.UpdateRunAbandonedReason, + Message: "The stages are aborted due to abandonment", + }) + meta.SetStatusCondition(&updateRunStatus.Conditions, metav1.Condition{ + Type: string(placementv1beta1.StagedUpdateRunConditionSucceeded), + Status: metav1.ConditionFalse, + ObservedGeneration: updateRun.GetGeneration(), + Reason: condition.UpdateRunAbandonedReason, + Message: "The update run has been abandoned", + }) + + if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { + klog.ErrorS(updateErr, "Failed to update the updateRun status as failed", "updateRun", klog.KObj(updateRun)) + // updateErr can be retried. + return controller.NewUpdateIgnoreConflictError(updateErr) + } + return nil +} + // recordUpdateRunStatus records the updateRun status. func (r *Reconciler) recordUpdateRunStatus(ctx context.Context, updateRun placementv1beta1.UpdateRunObj) error { if updateErr := r.Client.Status().Update(ctx, updateRun); updateErr != nil { diff --git a/pkg/controllers/updaterun/controller_integration_test.go b/pkg/controllers/updaterun/controller_integration_test.go index 8c8e38d06..3d16e3a02 100644 --- a/pkg/controllers/updaterun/controller_integration_test.go +++ b/pkg/controllers/updaterun/controller_integration_test.go @@ -272,6 +272,16 @@ func generateMetricsLabels( } } +func generateInitializationSucceededMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), + string(metav1.ConditionTrue), condition.UpdateRunInitializeSucceededReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + func generateInitializationFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { return &prometheusclientmodel.Metric{ Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionInitialized), @@ -312,6 +322,26 @@ func generateStuckMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *pr } } +func generatePausedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionProgressing), + string(metav1.ConditionFalse), condition.UpdateRunPausedReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + +func generateAbandonedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { + return &prometheusclientmodel.Metric{ + Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionSucceeded), + string(metav1.ConditionFalse), condition.UpdateRunAbandonedReason), + Gauge: &prometheusclientmodel.Gauge{ + Value: ptr.To(float64(time.Now().UnixNano()) / 1e9), + }, + } +} + func generateFailedMetric(updateRun *placementv1beta1.ClusterStagedUpdateRun) *prometheusclientmodel.Metric { return &prometheusclientmodel.Metric{ Label: generateMetricsLabels(updateRun, string(placementv1beta1.StagedUpdateRunConditionSucceeded), @@ -341,6 +371,7 @@ func generateTestClusterStagedUpdateRun() *placementv1beta1.ClusterStagedUpdateR PlacementName: testCRPName, ResourceSnapshotIndex: testResourceSnapshotIndex, StagedUpdateStrategyName: testUpdateStrategyName, + State: placementv1beta1.StateStarted, }, } } @@ -796,23 +827,14 @@ func generateFalseCondition(obj client.Object, condType any) metav1.Condition { } } -func generateFalseProgressingCondition(obj client.Object, condType any, succeeded bool) metav1.Condition { +func generateFalseProgressingCondition(obj client.Object, condType any, reason string) metav1.Condition { + falseCond := generateFalseCondition(obj, condType) + falseCond.Reason = reason + return falseCond +} + +func generateFalseSucceededCondition(obj client.Object, condType any, reason string) metav1.Condition { falseCond := generateFalseCondition(obj, condType) - reason := "" - switch condType { - case placementv1beta1.StagedUpdateRunConditionProgressing: - if succeeded { - reason = condition.UpdateRunSucceededReason - } else { - reason = condition.UpdateRunFailedReason - } - case placementv1beta1.StageUpdatingConditionProgressing: - if succeeded { - reason = condition.StageUpdatingSucceededReason - } else { - reason = condition.StageUpdatingFailedReason - } - } falseCond.Reason = reason return falseCond } diff --git a/pkg/controllers/updaterun/execution.go b/pkg/controllers/updaterun/execution.go index 198980749..3b6c8a784 100644 --- a/pkg/controllers/updaterun/execution.go +++ b/pkg/controllers/updaterun/execution.go @@ -241,31 +241,46 @@ func (r *Reconciler) executeUpdatingStage( } if finishedClusterCount == len(updatingStageStatus.Clusters) { - // All the clusters in the stage have been updated. - markUpdateRunWaiting(updateRun, updatingStageStatus.StageName) - markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration()) - klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) - // Check if the after stage tasks are ready. - approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) - if err != nil { - return 0, err - } - if approved { - markUpdateRunProgressing(updateRun) - markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) - // No need to wait to get to the next stage. - return 0, nil - } - // The after stage tasks are not ready yet. - if waitTime < 0 { - waitTime = stageUpdatingWaitTime - } - return waitTime, nil + return r.handleStageCompletion(ctx, updatingStageIndex, updateRun, updatingStageStatus) } + // Some clusters are still updating. return clusterUpdatingWaitTime, nil } +// handleStageCompletion handles the completion logic when all clusters in a stage are finished. +// Returns the wait time and any error encountered. +func (r *Reconciler) handleStageCompletion( + ctx context.Context, + updatingStageIndex int, + updateRun placementv1beta1.UpdateRunObj, + updatingStageStatus *placementv1beta1.StageUpdatingStatus, +) (time.Duration, error) { + updateRunRef := klog.KObj(updateRun) + + // All the clusters in the stage have been updated. + markUpdateRunWaiting(updateRun, updatingStageStatus.StageName) + markStageUpdatingWaiting(updatingStageStatus, updateRun.GetGeneration()) + klog.V(2).InfoS("The stage has finished all cluster updating", "stage", updatingStageStatus.StageName, "updateRun", updateRunRef) + + // Check if the after stage tasks are ready. + approved, waitTime, err := r.checkAfterStageTasksStatus(ctx, updatingStageIndex, updateRun) + if err != nil { + return 0, err + } + if approved { + markUpdateRunProgressing(updateRun) + markStageUpdatingSucceeded(updatingStageStatus, updateRun.GetGeneration()) + // No need to wait to get to the next stage. + return 0, nil + } + // The after stage tasks are not ready yet. + if waitTime < 0 { + waitTime = stageUpdatingWaitTime + } + return waitTime, nil +} + // executeDeleteStage executes the delete stage by deleting the bindings. func (r *Reconciler) executeDeleteStage( ctx context.Context, @@ -293,7 +308,8 @@ func (r *Reconciler) executeDeleteStage( // In validation, we already check the binding must exist in the status. delete(existingDeleteStageClusterMap, bindingSpec.TargetCluster) // Make sure the cluster is not marked as deleted as the binding is still there. - if condition.IsConditionStatusTrue(meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)), updateRun.GetGeneration()) { + clusterDeleteSucceededCond := meta.FindStatusCondition(curCluster.Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)) + if clusterDeleteSucceededCond != nil && clusterDeleteSucceededCond.Status == metav1.ConditionTrue { unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("the deleted cluster `%s` in the deleting stage still has a binding", bindingSpec.TargetCluster)) klog.ErrorS(unexpectedErr, "The cluster in the deleting stage is not removed yet but marked as deleted", "cluster", curCluster.ClusterName, "updateRun", updateRunRef) return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error()) diff --git a/pkg/controllers/updaterun/execution_integration_test.go b/pkg/controllers/updaterun/execution_integration_test.go index 980283497..202188460 100644 --- a/pkg/controllers/updaterun/execution_integration_test.go +++ b/pkg/controllers/updaterun/execution_integration_test.go @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + io_prometheus_client "github.com/prometheus/client_model/go" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -293,7 +294,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) // 1st stage completed, mark progressing condition reason as succeeded and add succeeded condition. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // 2nd stage started. wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) @@ -441,7 +442,7 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[1].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[1].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[1].Conditions = append(wantStatus.StagesStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) wantStatus.Conditions[1] = generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing) @@ -498,10 +499,10 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { wantStatus.DeletionStageStatus.Clusters[i].Conditions = append(wantStatus.DeletionStageStatus.Clusters[i].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) } // Mark the stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.DeletionStageStatus.Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -566,9 +567,9 @@ var _ = Describe("UpdateRun execution tests - double stages", func() { By("Validating the updateRun has failed") wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, false) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingFailedReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateFalseCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunFailedReason) wantStatus.Conditions = append(wantStatus.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -739,13 +740,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -845,13 +846,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -977,13 +978,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionApprovalRequestApproved)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1073,13 +1074,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 3rd cluster has succeeded and stage waiting for AfterStageTasks") wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) // 1st stage completed. - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") @@ -1271,13 +1272,13 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { By("Validating the 1st stage has completed") wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions = append(wantStatus.StagesStatus[0].AfterStageTaskStatus[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageTaskConditionWaitTimeElapsed)) - wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true) + wantStatus.StagesStatus[0].Conditions[0] = generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason) wantStatus.StagesStatus[0].Conditions = append(wantStatus.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark the deletion stage progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, true)) + wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing, condition.StageUpdatingSucceededReason)) wantStatus.DeletionStageStatus.Conditions = append(wantStatus.DeletionStageStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionSucceeded)) // Mark updateRun progressing condition as false with succeeded reason and add succeeded condition. - wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, true) + wantStatus.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunSucceededReason) wantStatus.Conditions = append(wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) // Need to have a longer wait time for the test to pass, because of the long wait time specified in the update strategy. timeout = time.Second * 90 @@ -1304,6 +1305,394 @@ var _ = Describe("UpdateRun execution tests - single stage", func() { }, timeout, interval).Should(BeTrue(), "failed to ensure the approvalRequest is not recreated") }) }) + + Context("Cluster staged update run should NOT update clusters one by one - different states (NotStarted -> Abandoned)", Ordered, func() { + var wantMetrics []*io_prometheus_client.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateNotStarted + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and but not execution started") + wantStatus = generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateInitializationSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start execution when the state is NotStarted", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + return meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) == nil && + meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) == nil + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + + By("Validating the 1st clusterResourceBinding is updated to NOT Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBindingState(ctx, binding) + }) + + It("Should not continue further after changing the state to Abandoned", func() { + By("Updating the updateRun state to Abandoned") + updateRun.Spec.State = placementv1beta1.StateAbandoned + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has not started") + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) + + Context("Cluster staged update run should update clusters one by one - different states (NotStarted -> Started-> Abandoned)", Ordered, func() { + var wantMetrics []*io_prometheus_client.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateNotStarted + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and but not execution started") + wantStatus = generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateInitializationSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start execution when the state is NotStarted", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + return meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) == nil && + meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) == nil + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + + By("Validating the 1st clusterResourceBinding is updated to NOT Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBindingState(ctx, binding) + }) + + It("Should start execution after changing the state to Started", func() { + By("Updating the updateRun state to Started") + updateRun.Spec.State = placementv1beta1.StateStarted + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has started") + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should abandon execution after changing the state to Abandoned", func() { + By("Updating the updateRun state to Abandoned") + updateRun.Spec.State = placementv1beta1.StateAbandoned + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has been abandoned") + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunAbandonedReason)) + wantStatus.Conditions = append(wantStatus.Conditions, generateFalseSucceededCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded, condition.UpdateRunAbandonedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateAbandonedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) + + Context("Cluster staged update run should update clusters one by one - different states (NotStarted -> Started -> Stopped -> Abandoned)", Ordered, func() { + var wantMetrics []*io_prometheus_client.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateNotStarted + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and but not execution started") + wantStatus = generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateInitializationSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start execution when the state is NotStarted", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + return meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) == nil && + meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) == nil + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + + By("Validating the 1st clusterResourceBinding is updated to NOT Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBindingState(ctx, binding) + }) + + It("Should start execution after changing the state to Started", func() { + By("Updating the updateRun state to Started") + updateRun.Spec.State = placementv1beta1.StateStarted + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has started") + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should stop execution after changing the state to Stopped", func() { + By("Updating the updateRun state to Stopped") + updateRun.Spec.State = placementv1beta1.StateStopped + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has stopped at the 2nd cluster of the 1st stage") + generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionInitialized) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunPausedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generatePausedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not continue execution when the state is Stopped", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + updateRunStatusCond := meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + return condition.IsConditionStatusFalse(updateRunStatusCond, currentUpdateRun.Generation) + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + }) + + It("Should abandon execution after changing the state to Abandoned", func() { + By("Updating the updateRun state to Abandoned") + updateRun.Spec.State = placementv1beta1.StateAbandoned + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has been abandoned") + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunAbandonedReason)) + wantStatus.Conditions = append(wantStatus.Conditions, generateFalseSucceededCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded, condition.UpdateRunAbandonedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateAbandonedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) + + Context("Cluster staged update run should update clusters one by one - different states (NotStarted -> Started -> Stopped -> Started -> Abandoned)", Ordered, func() { + var wantMetrics []*io_prometheus_client.Metric + BeforeAll(func() { + By("Creating a new clusterStagedUpdateRun") + updateRun.Spec.State = placementv1beta1.StateNotStarted + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the initialization succeeded and but not execution started") + wantStatus = generateSucceededInitializationStatusForSmallClusters(crp, updateRun, testResourceSnapshotIndex, policySnapshot, updateStrategy) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateInitializationSucceededMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not start execution when the state is NotStarted", func() { + By("Validating no execution has started") + Consistently(func() bool { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return false + } + return meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) == nil && + meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) == nil + }, timeout, interval).Should(BeTrue(), "execution has started unexpectedly") + + By("Validating the 1st clusterResourceBinding is updated to NOT Bound") + binding := resourceBindings[0] // cluster-0 + validateNotBindingState(ctx, binding) + }) + + It("Should start execution after changing the state to Started", func() { + By("Updating the updateRun state to Started") + updateRun.Spec.State = placementv1beta1.StateStarted + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has started") + wantStatus = generateExecutionStartedStatus(updateRun, wantStatus) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 1st cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 1st clusterResourceBinding is updated to Bound") + binding := resourceBindings[0] // cluster-0 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 1st clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 1st cluster has succeeded and 2nd cluster has started") + wantStatus.StagesStatus[0].Clusters[0].Conditions = append(wantStatus.StagesStatus[0].Clusters[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Validating the 1st stage has startTime set") + Expect(updateRun.Status.StagesStatus[0].StartTime).ShouldNot(BeNil()) + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should stop execution after changing the state to Stopped", func() { + By("Updating the updateRun state to Stopped") + updateRun.Spec.State = placementv1beta1.StateStopped + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has stopped at the 2nd cluster of the 1st stage") + generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionInitialized) + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunPausedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generatePausedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should not continue execution when the state is Stopped", func() { + By("Validating no execution has started") + Consistently(func() error { + var currentUpdateRun placementv1beta1.ClusterStagedUpdateRun + if err := k8sClient.Get(ctx, types.NamespacedName{Name: updateRun.Name}, ¤tUpdateRun); err != nil { + return err + } + updateRunStatusCond := meta.FindStatusCondition(currentUpdateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionProgressing)) + if condition.IsConditionStatusTrue(updateRunStatusCond, currentUpdateRun.Generation) { + return fmt.Errorf("update run progressing condition is true unexpectedly") + } + updateRunClusterStatusCond := meta.FindStatusCondition(currentUpdateRun.Status.StagesStatus[0].Clusters[1].Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)) + if updateRunClusterStatusCond != nil { + return fmt.Errorf("2nd cluster in 1st stage succeeded condition is set unexpectedly") + } + return nil + }, timeout, interval).Should(BeNil(), "execution has started unexpectedly") + }) + + It("Should continue execution after changing the state to Started", func() { + By("Updating the updateRun state to Started") + updateRun.Spec.State = placementv1beta1.StateStarted + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has started") + // UpdateRun is already initialized, so only need to set the progressing condition to true. + meta.SetStatusCondition(&wantStatus.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateProgressingMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should mark the 2nd cluster in the 1st stage as succeeded after marking the binding available", func() { + By("Validating the 2nd clusterResourceBinding is updated to Bound") + binding := resourceBindings[1] // cluster-1 + validateBindingState(ctx, binding, resourceSnapshot.Name, updateRun, 0) + + By("Updating the 2nd clusterResourceBinding to Available") + meta.SetStatusCondition(&binding.Status.Conditions, generateTrueCondition(binding, placementv1beta1.ResourceBindingAvailable)) + Expect(k8sClient.Status().Update(ctx, binding)).Should(Succeed(), "failed to update the binding status") + + By("Validating the 2nd cluster has succeeded and 3rd cluster has started") + wantStatus.StagesStatus[0].Clusters[1].Conditions = append(wantStatus.StagesStatus[0].Clusters[1].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionSucceeded)) + wantStatus.StagesStatus[0].Clusters[2].Conditions = append(wantStatus.StagesStatus[0].Clusters[2].Conditions, generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + + It("Should abandon execution after changing the state to Abandoned", func() { + By("Updating the updateRun state to Abandoned") + updateRun.Spec.State = placementv1beta1.StateAbandoned + Expect(k8sClient.Update(ctx, updateRun)).Should(Succeed(), "failed to update the updateRun state") + + By("Validating the execution has been abandoned") + meta.SetStatusCondition(&wantStatus.Conditions, generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunAbandonedReason)) + wantStatus.Conditions = append(wantStatus.Conditions, generateFalseSucceededCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded, condition.UpdateRunAbandonedReason)) + validateClusterStagedUpdateRunStatus(ctx, updateRun, wantStatus, "") + + By("Checking update run status metrics are emitted") + wantMetrics = append(wantMetrics, generateAbandonedMetric(updateRun)) + validateUpdateRunMetricsEmitted(wantMetrics...) + }) + }) }) func validateBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding, resourceSnapshotName string, updateRun *placementv1beta1.ClusterStagedUpdateRun, stage int) { @@ -1336,6 +1725,24 @@ func validateBindingState(ctx context.Context, binding *placementv1beta1.Cluster }, timeout, interval).Should(Succeed(), "failed to validate the binding state") } +func validateNotBindingState(ctx context.Context, binding *placementv1beta1.ClusterResourceBinding) { + Eventually(func() error { + if err := k8sClient.Get(ctx, types.NamespacedName{Name: binding.Name}, binding); err != nil { + return err + } + + if binding.Spec.State == placementv1beta1.BindingStateBound { + return fmt.Errorf("binding %s is in Bound state, got %s", binding.Name, binding.Spec.State) + } + + rolloutStartedCond := binding.GetCondition(string(placementv1beta1.ResourceBindingRolloutStarted)) + if condition.IsConditionStatusTrue(rolloutStartedCond, binding.Generation) { + return fmt.Errorf("binding %s does have RolloutStarted condition", binding.Name) + } + return nil + }, timeout, interval).Should(Succeed(), "failed to validate the not binding state") +} + func approveClusterApprovalRequest(ctx context.Context, approvalRequestName string) { Eventually(func() error { var approvalRequest placementv1beta1.ClusterApprovalRequest diff --git a/pkg/controllers/updaterun/initialization_integration_test.go b/pkg/controllers/updaterun/initialization_integration_test.go index 9a4507eee..86b8717fa 100644 --- a/pkg/controllers/updaterun/initialization_integration_test.go +++ b/pkg/controllers/updaterun/initialization_integration_test.go @@ -941,6 +941,56 @@ var _ = Describe("Updaterun initialization tests", func() { validateUpdateRunMetricsEmitted(generateProgressingMetric(updateRun)) }) }) + + It("Should not initialize if updateRun is created with state Abandoned", func() { + By("Creating a new clusterStagedUpdateRun in Abandoned state") + updateRun.Spec.State = placementv1beta1.StateAbandoned + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the updateRun is not initialized") + // Populate the cache first. + Eventually(func() error { + if err := k8sClient.Get(ctx, updateRunNamespacedName, updateRun); err != nil { + return err + } + return nil + }, timeout, interval).Should(Succeed(), "failed to get the updateRun") + Consistently(func() error { + if err := k8sClient.Get(ctx, updateRunNamespacedName, updateRun); err != nil { + return err + } + initCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) + if initCond != nil { + return fmt.Errorf("got initialization condition: %v, want nil", initCond) + } + return nil + }, duration, interval).Should(Succeed(), "the abandoned updateRun should not be initialized") + }) + + It("Should not initialize if updateRun is created with state Stopped ", func() { + By("Creating a new clusterStagedUpdateRun in Stopped state") + updateRun.Spec.State = placementv1beta1.StateStopped + Expect(k8sClient.Create(ctx, updateRun)).To(Succeed()) + + By("Validating the updateRun is not initialized") + // Populate the cache first. + Eventually(func() error { + if err := k8sClient.Get(ctx, updateRunNamespacedName, updateRun); err != nil { + return err + } + return nil + }, timeout, interval).Should(Succeed(), "failed to get the updateRun") + Consistently(func() error { + if err := k8sClient.Get(ctx, updateRunNamespacedName, updateRun); err != nil { + return err + } + initCond := meta.FindStatusCondition(updateRun.Status.Conditions, string(placementv1beta1.StagedUpdateRunConditionInitialized)) + if initCond != nil { + return fmt.Errorf("got initialization condition: %v, want nil", initCond) + } + return nil + }, duration, interval).Should(Succeed(), "the stopped updateRun should not be initialized") + }) }) func validateFailedInitCondition(ctx context.Context, updateRun *placementv1beta1.ClusterStagedUpdateRun, message string) { @@ -1071,13 +1121,13 @@ func generateSucceededInitializationStatusForSmallClusters( func generateExecutionStartedStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, - initialized *placementv1beta1.UpdateRunStatus, + status *placementv1beta1.UpdateRunStatus, ) *placementv1beta1.UpdateRunStatus { // Mark updateRun execution has started. - initialized.Conditions = append(initialized.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) + meta.SetStatusCondition(&status.Conditions, generateTrueCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing)) // Mark updateRun 1st stage has started. - initialized.StagesStatus[0].Conditions = append(initialized.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) + meta.SetStatusCondition(&status.StagesStatus[0].Conditions, generateTrueCondition(updateRun, placementv1beta1.StageUpdatingConditionProgressing)) // Mark updateRun 1st cluster in the 1st stage has started. - initialized.StagesStatus[0].Clusters[0].Conditions = []metav1.Condition{generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)} - return initialized + status.StagesStatus[0].Clusters[0].Conditions = []metav1.Condition{generateTrueCondition(updateRun, placementv1beta1.ClusterUpdatingConditionStarted)} + return status } diff --git a/pkg/controllers/updaterun/validation.go b/pkg/controllers/updaterun/validation.go index ffa5ea3c2..c92a946a9 100644 --- a/pkg/controllers/updaterun/validation.go +++ b/pkg/controllers/updaterun/validation.go @@ -22,6 +22,7 @@ import ( "reflect" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" @@ -185,7 +186,7 @@ func validateClusterUpdatingStatus( ) (int, int, error) { stageSucceedCond := meta.FindStatusCondition(stageStatus.Conditions, string(placementv1beta1.StageUpdatingConditionSucceeded)) stageStartedCond := meta.FindStatusCondition(stageStatus.Conditions, string(placementv1beta1.StageUpdatingConditionProgressing)) - if condition.IsConditionStatusTrue(stageSucceedCond, updateRun.GetGeneration()) { + if stageSucceedCond != nil && stageSucceedCond.Status == metav1.ConditionTrue { // The stage has finished. if updatingStageIndex != -1 && curStage > updatingStageIndex { // The finished stage is after the updating stage. @@ -196,10 +197,8 @@ func validateClusterUpdatingStatus( // Make sure that all the clusters are updated. for curCluster := range stageStatus.Clusters { // Check if the cluster is still updating. - if !condition.IsConditionStatusTrue(meta.FindStatusCondition( - stageStatus.Clusters[curCluster].Conditions, - string(placementv1beta1.ClusterUpdatingConditionSucceeded)), - updateRun.GetGeneration()) { + clusterSucceededCond := meta.FindStatusCondition(stageStatus.Clusters[curCluster].Conditions, string(placementv1beta1.ClusterUpdatingConditionSucceeded)) + if clusterSucceededCond == nil || clusterSucceededCond.Status == metav1.ConditionFalse { // The clusters in the finished stage should all have finished too. unexpectedErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("cluster `%s` in the finished stage `%s` has not succeeded", stageStatus.Clusters[curCluster].ClusterName, stageStatus.StageName)) klog.ErrorS(unexpectedErr, "The cluster in a finished stage is still updating", "updateRun", klog.KObj(updateRun)) @@ -214,7 +213,7 @@ func validateClusterUpdatingStatus( } // Record the last finished stage so we can continue from the next stage if no stage is updating. lastFinishedStageIndex = curStage - } else if condition.IsConditionStatusFalse(stageSucceedCond, updateRun.GetGeneration()) { + } else if stageSucceedCond != nil && stageSucceedCond.Status == metav1.ConditionFalse { // The stage has failed. failedErr := fmt.Errorf("the stage `%s` has failed, err: %s", stageStatus.StageName, stageSucceedCond.Message) klog.ErrorS(failedErr, "The stage has failed", "stageCond", stageSucceedCond, "updateRun", klog.KObj(updateRun)) diff --git a/pkg/controllers/updaterun/validation_integration_test.go b/pkg/controllers/updaterun/validation_integration_test.go index bcc255473..4c307a691 100644 --- a/pkg/controllers/updaterun/validation_integration_test.go +++ b/pkg/controllers/updaterun/validation_integration_test.go @@ -34,6 +34,7 @@ import ( clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1" placementv1beta1 "github.com/kubefleet-dev/kubefleet/apis/placement/v1beta1" "github.com/kubefleet-dev/kubefleet/pkg/utils" + "github.com/kubefleet-dev/kubefleet/pkg/utils/condition" ) var _ = Describe("UpdateRun validation tests", func() { @@ -564,7 +565,7 @@ func generateFailedValidationStatus( updateRun *placementv1beta1.ClusterStagedUpdateRun, started *placementv1beta1.UpdateRunStatus, ) *placementv1beta1.UpdateRunStatus { - started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, false) + started.Conditions[1] = generateFalseProgressingCondition(updateRun, placementv1beta1.StagedUpdateRunConditionProgressing, condition.UpdateRunFailedReason) started.Conditions = append(started.Conditions, generateFalseCondition(updateRun, placementv1beta1.StagedUpdateRunConditionSucceeded)) return started } diff --git a/pkg/utils/condition/reason.go b/pkg/utils/condition/reason.go index b1d963e58..db4672c9c 100644 --- a/pkg/utils/condition/reason.go +++ b/pkg/utils/condition/reason.go @@ -161,9 +161,15 @@ const ( // UpdateRunProgressingReason is the reason string of condition if the staged update run is progressing. UpdateRunProgressingReason = "UpdateRunProgressing" + // UpdateRunPausedReason is the reason string of condition if the staged update run is paused. + UpdateRunPausedReason = "UpdateRunPaused" + // UpdateRunFailedReason is the reason string of condition if the staged update run failed. UpdateRunFailedReason = "UpdateRunFailed" + // UpdateRunAbandonedReason is the reason string of condition if the staged update run is abandoned. + UpdateRunAbandonedReason = "UpdateRunAbandoned" + // UpdateRunStuckReason is the reason string of condition if the staged update run is stuck waiting for a cluster to be updated. UpdateRunStuckReason = "UpdateRunStuck" diff --git a/test/e2e/actuals_test.go b/test/e2e/actuals_test.go index 344faf6f4..904440b6d 100644 --- a/test/e2e/actuals_test.go +++ b/test/e2e/actuals_test.go @@ -2040,6 +2040,17 @@ func updateRunStageRolloutSucceedConditions(generation int64) []metav1.Condition } } +func updateRunStageRolloutWaitingConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StageUpdatingConditionProgressing), + Status: metav1.ConditionFalse, + Reason: condition.StageUpdatingWaitingReason, + ObservedGeneration: generation, + }, + } +} + func updateRunAfterStageTaskSucceedConditions(generation int64, taskType placementv1beta1.StageTaskType) []metav1.Condition { if taskType == placementv1beta1.StageTaskTypeApproval { return []metav1.Condition{ @@ -2090,6 +2101,57 @@ func updateRunSucceedConditions(generation int64) []metav1.Condition { } } +func updateRunStoppedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunInitializeSucceededReason, + ObservedGeneration: 1, + }, + { + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionFalse, + Reason: condition.UpdateRunPausedReason, + ObservedGeneration: generation, + }, + } +} + +func updateRunAbandonedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunInitializeSucceededReason, + ObservedGeneration: 1, + }, + { + Type: string(placementv1beta1.StagedUpdateRunConditionProgressing), + Status: metav1.ConditionFalse, + Reason: condition.UpdateRunAbandonedReason, + ObservedGeneration: generation, + }, + { + Type: string(placementv1beta1.StagedUpdateRunConditionSucceeded), + Status: metav1.ConditionFalse, + Reason: condition.UpdateRunAbandonedReason, + ObservedGeneration: generation, + }, + } +} + +func updateRunInitializedConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + { + Type: string(placementv1beta1.StagedUpdateRunConditionInitialized), + Status: metav1.ConditionTrue, + Reason: condition.UpdateRunInitializeSucceededReason, + ObservedGeneration: generation, + }, + } +} + func clusterStagedUpdateRunStatusSucceededActual( updateRunName string, wantResourceIndex string, @@ -2126,6 +2188,256 @@ func clusterStagedUpdateRunStatusSucceededActual( } } +func clusterStagedUpdateRunStatusAbandonedActual( + updateRunName string, + wantResourceIndex string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + wantStatus.StagesStatus = buildStageUpdatingStatusesWithGeneration(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun, updateRun.GetGeneration()-2) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunAbandonedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) + } + return nil + } +} + +func clusterStagedUpdateRunStatusStoppedActual( + updateRunName string, + wantResourceIndex string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + wantStatus.StagesStatus = buildStageUpdatingStatusesWithGeneration(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun, updateRun.GetGeneration()-1) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunStoppedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) + } + return nil + } +} + +func clusterStagedUpdateRunStatusNotStartedActual( + updateRunName string, + wantResourceIndex string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } + } + wantStatus.StagesStatus = stagesStatus + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunInitializedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) + } + return nil + } +} + +func stagedUpdateRunStatusAbandonedActual( + updateRunName, namespace string, + wantResourceIndex string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.StagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName, Namespace: namespace}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + wantStatus.StagesStatus = buildStageUpdatingStatusesWithGeneration(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun, updateRun.GetGeneration()-2) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunAbandonedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) + } + return nil + } +} + +func stagedUpdateRunStatusNotStartedActual( + updateRunName, namespace string, + wantResourceIndex string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.StagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName, Namespace: namespace}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + } + } + wantStatus.StagesStatus = stagesStatus + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunInitializedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) + } + return nil + } +} + +func stagedUpdateRunStatusStoppedActual( + updateRunName, namespace string, + wantResourceIndex string, + wantPolicyIndex string, + wantClusterCount int, + wantApplyStrategy *placementv1beta1.ApplyStrategy, + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantUnscheduledClusters []string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, +) func() error { + return func() error { + updateRun := &placementv1beta1.StagedUpdateRun{} + if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName, Namespace: namespace}, updateRun); err != nil { + return err + } + + wantStatus := placementv1beta1.UpdateRunStatus{ + PolicySnapshotIndexUsed: wantPolicyIndex, + ResourceSnapshotIndexUsed: wantResourceIndex, + PolicyObservedClusterCount: wantClusterCount, + ApplyStrategy: wantApplyStrategy.DeepCopy(), + UpdateStrategySnapshot: wantStrategySpec, + } + + wantStatus.StagesStatus = buildStageUpdatingStatusesWithGeneration(wantStrategySpec, wantSelectedClusters, wantCROs, wantROs, updateRun, updateRun.GetGeneration()-1) + wantStatus.DeletionStageStatus = buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + wantStatus.Conditions = updateRunStoppedConditions(updateRun.Generation) + if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" { + return fmt.Errorf("UpdateRun status diff (-got, +want): %s", diff) + } + return nil + } +} + func stagedUpdateRunStatusSucceededActual( updateRunName, namespace string, wantResourceIndex, wantPolicyIndex string, @@ -2161,6 +2473,58 @@ func stagedUpdateRunStatusSucceededActual( } } +func buildStageUpdatingStatusesWithGeneration( + wantStrategySpec *placementv1beta1.UpdateStrategySpec, + wantSelectedClusters [][]string, + wantCROs map[string][]string, + wantROs map[string][]placementv1beta1.NamespacedName, + updateRun placementv1beta1.UpdateRunObj, + generation int64, +) []placementv1beta1.StageUpdatingStatus { + stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages)) + for i, stage := range wantStrategySpec.Stages { + stagesStatus[i].StageName = stage.Name + stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i])) + for j := range stagesStatus[i].Clusters { + stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j] + stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]] + stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]] + if i == 0 { + stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(generation) + } + } + stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.StageTaskStatus, len(stage.AfterStageTasks)) + for j, task := range stage.AfterStageTasks { + stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.GetName(), stage.Name) + } + if i == 0 { + if task.Type == placementv1beta1.StageTaskTypeApproval { + stagesStatus[i].AfterStageTaskStatus[j].Conditions = append(stagesStatus[i].AfterStageTaskStatus[j].Conditions, metav1.Condition{ + Type: string(placementv1beta1.StageTaskConditionApprovalRequestCreated), + Status: metav1.ConditionTrue, + Reason: condition.AfterStageTaskApprovalRequestCreatedReason, + ObservedGeneration: generation, + }) + } + if task.Type == placementv1beta1.StageTaskTypeTimedWait { + stagesStatus[i].AfterStageTaskStatus[j].Conditions = append(stagesStatus[i].AfterStageTaskStatus[j].Conditions, metav1.Condition{ + Type: string(placementv1beta1.StageTaskConditionWaitTimeElapsed), + Status: metav1.ConditionTrue, + Reason: condition.AfterStageTaskWaitTimeElapsedReason, + ObservedGeneration: generation, + }) + } + } + } + if i == 0 { + stagesStatus[i].Conditions = updateRunStageRolloutWaitingConditions(generation) + } + } + return stagesStatus +} + func buildStageUpdatingStatuses( wantStrategySpec *placementv1beta1.UpdateStrategySpec, wantSelectedClusters [][]string, @@ -2194,6 +2558,15 @@ func buildStageUpdatingStatuses( func buildDeletionStageStatus( wantUnscheduledClusters []string, updateRun placementv1beta1.UpdateRunObj, +) *placementv1beta1.StageUpdatingStatus { + deleteStageStatus := buildDeletionStatusWithoutConditions(wantUnscheduledClusters, updateRun) + deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) + return deleteStageStatus +} + +func buildDeletionStatusWithoutConditions( + wantUnscheduledClusters []string, + updateRun placementv1beta1.UpdateRunObj, ) *placementv1beta1.StageUpdatingStatus { deleteStageStatus := &placementv1beta1.StageUpdatingStatus{ StageName: "kubernetes-fleet.io/deleteStage", @@ -2203,7 +2576,6 @@ func buildDeletionStageStatus( deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i] deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.GetGeneration()) } - deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.GetGeneration()) return deleteStageStatus } diff --git a/test/e2e/cluster_staged_updaterun_test.go b/test/e2e/cluster_staged_updaterun_test.go index 8e8b28822..a7fd810aa 100644 --- a/test/e2e/cluster_staged_updaterun_test.go +++ b/test/e2e/cluster_staged_updaterun_test.go @@ -27,6 +27,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -147,10 +148,6 @@ var _ = Describe("test CRP rollout with staged update run", func() { validateAndApproveClusterApprovalRequests(updateRunNames[0], envCanary) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) - }) - It("Should rollout resources to all the members and complete the cluster staged update run successfully", func() { csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) @@ -301,7 +298,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -363,7 +360,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -401,7 +398,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -504,7 +501,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -552,7 +549,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -601,7 +598,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -694,7 +691,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -741,7 +738,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -789,7 +786,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[2], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -959,7 +956,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -1062,7 +1059,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1176,7 +1173,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateStarted) // Verify rollout to canary cluster first By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1246,7 +1243,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { It("Should create a staged update run and verify cluster approval request is created", func() { validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) // Verify that cluster approval request is created for canary stage. Eventually(func() error { @@ -1381,7 +1378,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Create updateRun and verify resources are rolled out", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) validateAndApproveClusterApprovalRequests(updateRunName, envCanary) @@ -1513,7 +1510,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the cluster staged update run with all 3 clusters updated in parallel", func() { @@ -1603,7 +1600,7 @@ var _ = Describe("test CRP rollout with staged update run", func() { }) It("Should create a cluster staged update run successfully", func() { - createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the cluster staged update run with all 3 clusters", func() { @@ -1621,6 +1618,130 @@ var _ = Describe("test CRP rollout with staged update run", func() { Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) }) }) + + Context("Test resource rollout with staged update run by update run states - (NotStarted -> Started -> Stopped -> Abandoned)", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.ClusterStagedUpdateStrategy + + BeforeAll(func() { + // Create a test namespace and a configMap inside it on the hub cluster. + createWorkResources() + + // Create the CRP with external rollout strategy. + crp := &placementv1beta1.ClusterResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: crpName, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: workResourceSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP") + + // Create the clusterStagedUpdateStrategy. + strategy = createClusterStagedUpdateStrategySucceed(strategyName) + + for i := 0; i < 1; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(clusterStagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the CRP. + ensureCRPAndRelatedResourcesDeleted(crpName, allMemberClusters) + + // Remove all the clusterStagedUpdateRuns. + for _, name := range updateRunNames { + ensureClusterStagedUpdateRunDeletion(name) + } + + // Delete the clusterStagedUpdateStrategy. + ensureClusterUpdateRunStrategyDeletion(strategyName) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedWorkResourcesFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestClusterResourceSnapshot(crpName, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the crp", func() { + validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) + }) + + It("Should update crp status as pending rollout", func() { + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("Should create a cluster staged update run successfully", func() { + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateNotStarted) + }) + + It("Should not start rollout as the update run is in NotStarted state", func() { + By("Member clusters should not have work resources placed") + checkIfRemovedWorkResourcesFromAllMemberClustersConsistently() + + By("Validating the csur status remains in NotStarted state") + csurNotStartedActual := clusterStagedUpdateRunStatusNotStartedActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Consistently(csurNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep updateRun %s in NotStarted state", updateRunNames[0]) + }) + + It("Should rollout resources to member-cluster-2 only after update run is in Started state", func() { + // Update the update run state to Started. + By("Updating the update run state to Started") + updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateStarted) + + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + }) + + It("Should stop update run when updated to Stopped state", func() { + // Update the update run state to Stopped. + By("Updating the update run state to Stopped") + updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateStopped) + + By("Validating no further rollouts happen after stopping") + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated only") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + + csurSucceededActual := clusterStagedUpdateRunStatusStoppedActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + }) + + It("Should abandon update run when updated to Abandoned state", func() { + // Update the update run state to Abandoned. + By("Updating the update run state to Abandoned") + updateClusterStagedUpdateRunState(updateRunNames[0], placementv1beta1.StateAbandoned) + + By("Validating no further rollouts happen after abandonment") + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedWorkResourcesFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated only") + crpStatusUpdatedActual := crpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP %s status as expected", crpName) + + csurSucceededActual := clusterStagedUpdateRunStatusAbandonedActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(csurSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + }) + }) + + //TODO(britaniar): Add more e2e tests for updateRun Start/Stop Implementation }) // Note that this container cannot run in parallel with other containers. @@ -1688,7 +1809,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) By("Creating the first staged update run") - createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[0], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) By("Validating staged update run has succeeded") csurSucceededActual := clusterStagedUpdateRunStatusSucceededActual(updateRunNames[0], resourceSnapshotIndex1st, policySnapshotIndex1st, 3, defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[0], allMemberClusterNames[1], allMemberClusterNames[2]}}, nil, nil, nil) @@ -1739,7 +1860,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should create another staged update run for the same CRP", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 2) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the second staged update run and complete the CRP", func() { @@ -1787,7 +1908,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the staged update run, complete CRP, and rollout resources to all member clusters", func() { @@ -1830,7 +1951,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateStarted) }) It("Should complete the staged update run, complete CRP, and rollout updated resources to all member clusters", func() { @@ -1869,7 +1990,7 @@ var _ = Describe("Test member cluster join and leave flow with updateRun", Label It("Should reschedule to member cluster 1 and create a new cluster staged update run successfully", func() { validateLatestClusterSchedulingPolicySnapshot(crpName, policySnapshotIndex1st, 3) - createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName) + createClusterStagedUpdateRunSucceed(updateRunNames[1], crpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the staged update run, complete CRP, and re-place resources to all member clusters", func() { @@ -2010,12 +2131,13 @@ func validateLatestClusterResourceSnapshot(crpName, wantResourceSnapshotIndex st }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string) { +func createClusterStagedUpdateRunSucceed(updateRunName, crpName, resourceSnapshotIndex, strategyName string, state placementv1beta1.State) { updateRun := &placementv1beta1.ClusterStagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ + State: state, PlacementName: crpName, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, @@ -2030,6 +2152,7 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Name: updateRunName, }, Spec: placementv1beta1.UpdateRunSpec{ + State: placementv1beta1.StateStarted, PlacementName: crpName, StagedUpdateStrategyName: strategyName, }, @@ -2037,6 +2160,14 @@ func createClusterStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunNam Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create ClusterStagedUpdateRun %s", updateRunName) } +func updateClusterStagedUpdateRunState(updateRunName string, state placementv1beta1.State) { + updateRun := &placementv1beta1.ClusterStagedUpdateRun{} + Expect(hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun)).To(Succeed(), "Failed to get ClusterStagedUpdateRun %s", updateRunName) + + updateRun.Spec.State = state + Expect(hubClient.Update(ctx, updateRun)).To(Succeed(), "Failed to update ClusterStagedUpdateRun %s", updateRunName) +} + func validateAndApproveClusterApprovalRequests(updateRunName, stageName string) { Eventually(func() error { appReqList := &placementv1beta1.ClusterApprovalRequestList{} diff --git a/test/e2e/staged_updaterun_test.go b/test/e2e/staged_updaterun_test.go index 2f03affa9..5049e5654 100644 --- a/test/e2e/staged_updaterun_test.go +++ b/test/e2e/staged_updaterun_test.go @@ -138,10 +138,6 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem validateAndApproveNamespacedApprovalRequests(updateRunNames[0], testNamespace, envCanary) }) - It("Should rollout resources to member-cluster-1 first because of its name", func() { - checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[0]}) - }) - It("Should rollout resources to all the members and complete the staged update run successfully", func() { surSucceededActual := stagedUpdateRunStatusSucceededActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s/%s succeeded", testNamespace, updateRunNames[0]) @@ -289,7 +285,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -351,7 +347,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -389,7 +385,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a new staged update run with old resourceSnapshotIndex successfully to rollback", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollback resources to member-cluster-2 only and completes stage canary", func() { @@ -490,7 +486,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -538,7 +534,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on member-cluster-1 and member-cluster-2 only and completes stage canary", func() { @@ -587,7 +583,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -678,7 +674,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should not rollout any resources to member clusters and complete stage canary", func() { @@ -725,7 +721,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[1], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on member-cluster-2 and member-cluster-3 only and completes stage canary", func() { @@ -773,7 +769,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a namespaced staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunNames[2], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should still have resources on all member clusters and complete stage canary", func() { @@ -915,7 +911,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should rollout resources to member-cluster-2 only and complete stage canary", func() { @@ -1013,7 +1009,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should report diff for member-cluster-2 only and completes stage canary", func() { @@ -1125,7 +1121,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create a staged update run with new resourceSnapshotIndex and verify rollout happens", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex2nd, strategyName, placementv1beta1.StateStarted) // Verify rollout to canary cluster first. By("Verify that the new configmap is updated on member-cluster-2 during canary stage") @@ -1207,7 +1203,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Create updateRun and verify resources are rolled out", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) validateAndApproveNamespacedApprovalRequests(updateRunName, testNamespace, envCanary) @@ -1338,7 +1334,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the staged update run with all 3 clusters updated in parallel", func() { @@ -1427,7 +1423,7 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem }) It("Should create a staged update run successfully", func() { - createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName) + createStagedUpdateRunSucceed(updateRunName, testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateStarted) }) It("Should complete the staged update run with all 3 clusters", func() { @@ -1445,6 +1441,128 @@ var _ = Describe("test RP rollout with staged update run", Label("resourceplacem Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) }) }) + + Context("Test resource rollout with staged update run by update run states - (NotStarted -> Started -> Stopped -> Abandoned)", Ordered, func() { + updateRunNames := []string{} + var strategy *placementv1beta1.StagedUpdateStrategy + + BeforeAll(func() { + // Create the RP with external rollout strategy. + rp := &placementv1beta1.ResourcePlacement{ + ObjectMeta: metav1.ObjectMeta{ + Name: rpName, + Namespace: testNamespace, + // Add a custom finalizer; this would allow us to better observe + // the behavior of the controllers. + Finalizers: []string{customDeletionBlockerFinalizer}, + }, + Spec: placementv1beta1.PlacementSpec{ + ResourceSelectors: configMapSelector(), + Strategy: placementv1beta1.RolloutStrategy{ + Type: placementv1beta1.ExternalRolloutStrategyType, + }, + }, + } + Expect(hubClient.Create(ctx, rp)).To(Succeed(), "Failed to create RP") + + // Create the stagedUpdateStrategy. + strategy = createStagedUpdateStrategySucceed(strategyName, testNamespace) + + for i := 0; i < 3; i++ { + updateRunNames = append(updateRunNames, fmt.Sprintf(stagedUpdateRunNameWithSubIndexTemplate, GinkgoParallelProcess(), i)) + } + }) + + AfterAll(func() { + // Remove the custom deletion blocker finalizer from the RP. + ensureRPAndRelatedResourcesDeleted(types.NamespacedName{Name: rpName, Namespace: testNamespace}, allMemberClusters) + + // Remove all the stagedUpdateRuns. + for _, name := range updateRunNames { + ensureStagedUpdateRunDeletion(name, testNamespace) + } + + // Delete the stagedUpdateStrategy. + ensureStagedUpdateRunStrategyDeletion(strategyName, testNamespace) + }) + + It("Should not rollout any resources to member clusters as there's no update run yet", checkIfRemovedConfigMapFromAllMemberClustersConsistently) + + It("Should have the latest resource snapshot", func() { + validateLatestResourceSnapshot(rpName, testNamespace, resourceSnapshotIndex1st) + }) + + It("Should successfully schedule the rp", func() { + validateLatestSchedulingPolicySnapshot(rpName, testNamespace, policySnapshotIndex1st, 3) + }) + + It("Should update rp status as pending rollout", func() { + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", "", ""}, []bool{false, false, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s/%s status as expected", testNamespace, rpName) + }) + + It("Should create a staged update run successfully", func() { + createStagedUpdateRunSucceed(updateRunNames[0], testNamespace, rpName, resourceSnapshotIndex1st, strategyName, placementv1beta1.StateNotStarted) + }) + + It("Should not start rollout as the update run is in NotStarted state", func() { + By("Member clusters should not have work resources placed") + checkIfRemovedConfigMapFromAllMemberClustersConsistently() + + By("Validating the sur status remains in NotStarted state") + surNotStartedActual := stagedUpdateRunStatusNotStartedActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Consistently(surNotStartedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to keep updateRun %s in NotStarted state", updateRunNames[0]) + }) + + It("Should rollout resources to member-cluster-2 only after update run is in Started state", func() { + // Update the update run state to Started. + By("Updating the update run state to Started") + updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateStarted) + + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + }) + + It("Should stop update run when updated to Stopped state", func() { + // Update the update run state to Stopped. + By("Updating the update run state to Stopped") + updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateStopped) + + By("Validating no further rollouts happen after stopping") + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated only") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + + surSucceededActual := stagedUpdateRunStatusStoppedActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + }) + + It("Should abandon update run when updated to Abandoned state", func() { + // Update the update run state to Abandoned. + By("Updating the update run state to Abandoned") + updateStagedUpdateRunState(updateRunNames[0], testNamespace, placementv1beta1.StateAbandoned) + + By("Validating no further rollouts happen after abandonment") + checkIfPlacedWorkResourcesOnMemberClustersInUpdateRun([]*framework.Cluster{allMemberClusters[1]}) + checkIfRemovedConfigMapFromMemberClustersConsistently([]*framework.Cluster{allMemberClusters[0], allMemberClusters[2]}) + + By("Validating crp status as member-cluster-2 updated only") + rpStatusUpdatedActual := rpStatusWithExternalStrategyActual(nil, "", false, allMemberClusterNames, []string{"", resourceSnapshotIndex1st, ""}, []bool{false, true, false}, nil, nil) + Eventually(rpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update RP %s status as expected", rpName) + + surSucceededActual := stagedUpdateRunStatusAbandonedActual(updateRunNames[0], testNamespace, resourceSnapshotIndex1st, policySnapshotIndex1st, len(allMemberClusters), defaultApplyStrategy, &strategy.Spec, [][]string{{allMemberClusterNames[1]}, {allMemberClusterNames[0], allMemberClusterNames[2]}}, nil, nil, nil) + Eventually(surSucceededActual, updateRunEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to validate updateRun %s succeeded", updateRunNames[0]) + }) + }) + + //TODO(britaniar): Add more e2e tests for updateRun Start/Stop Implementation }) func createStagedUpdateStrategySucceed(strategyName, namespace string) *placementv1beta1.StagedUpdateStrategy { @@ -1535,7 +1653,7 @@ func validateLatestResourceSnapshot(rpName, namespace, wantResourceSnapshotIndex }, eventuallyDuration, eventuallyInterval).Should(Equal(wantResourceSnapshotIndex), "Resource snapshot index does not match") } -func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string) { +func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnapshotIndex, strategyName string, state placementv1beta1.State) { updateRun := &placementv1beta1.StagedUpdateRun{ ObjectMeta: metav1.ObjectMeta{ Name: updateRunName, @@ -1543,6 +1661,7 @@ func createStagedUpdateRunSucceed(updateRunName, namespace, rpName, resourceSnap }, Spec: placementv1beta1.UpdateRunSpec{ PlacementName: rpName, + State: state, ResourceSnapshotIndex: resourceSnapshotIndex, StagedUpdateStrategyName: strategyName, }, @@ -1557,6 +1676,7 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Namespace: namespace, }, Spec: placementv1beta1.UpdateRunSpec{ + State: placementv1beta1.StateStarted, PlacementName: rpName, StagedUpdateStrategyName: strategyName, }, @@ -1564,6 +1684,14 @@ func createStagedUpdateRunSucceedWithNoResourceSnapshotIndex(updateRunName, name Expect(hubClient.Create(ctx, updateRun)).To(Succeed(), "Failed to create StagedUpdateRun %s", updateRunName) } +func updateStagedUpdateRunState(updateRunName, namespace string, state placementv1beta1.State) { + updateRun := &placementv1beta1.StagedUpdateRun{} + Expect(hubClient.Get(ctx, types.NamespacedName{Name: updateRunName, Namespace: namespace}, updateRun)).To(Succeed(), "Failed to get StagedUpdateRun %s", updateRunName) + + updateRun.Spec.State = state + Expect(hubClient.Update(ctx, updateRun)).To(Succeed(), "Failed to update StagedUpdateRun %s", updateRunName) +} + func validateAndApproveNamespacedApprovalRequests(updateRunName, namespace, stageName string) { Eventually(func() error { appReqList := &placementv1beta1.ApprovalRequestList{}