diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index c3d1b39d0..4fba3ccf1 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -291,7 +291,7 @@ func (ctrl *resizeController) Run(workers int, ctx context.Context) { go ctrl.slowSet.Run(stopCh) } - for i := 0; i < workers; i++ { + for range workers { go wait.Until(ctrl.syncPVCs, 0, stopCh) } diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index b54e3d222..239fef686 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -211,7 +211,7 @@ func TestController(t *testing.T) { disableVolumeInUseErrorHandler: true, }, } { - client := csi.NewMockClient("mock", test.NodeResize, true, false, true, true, false) + client := csi.NewMockClient("mock", test.NodeResize, true, false, true, true) driverName, _ := client.GetDriverName(context.TODO()) var expectedCap resource.Quantity @@ -261,9 +261,7 @@ func TestController(t *testing.T) { stopCh := make(chan struct{}) informerFactory.Start(stopCh) - ctx := context.TODO() - defer ctx.Done() - go controller.Run(1, ctx) + go controller.Run(1, t.Context()) for _, obj := range initialObjects { switch obj.(type) { @@ -380,7 +378,7 @@ func TestResizePVC(t *testing.T) { }, } { t.Run(test.Name, func(t *testing.T) { - client := csi.NewMockClient("mock", test.NodeResize, true, false, true, true, false) + client := csi.NewMockClient("mock", test.NodeResize, true, false, true, true) if test.expansionError != nil { client.SetExpansionError(test.expansionError) } diff --git a/pkg/controller/expand_and_recover_test.go b/pkg/controller/expand_and_recover_test.go index 8d86d21e4..625359150 100644 --- a/pkg/controller/expand_and_recover_test.go +++ b/pkg/controller/expand_and_recover_test.go @@ -159,7 +159,7 @@ func TestExpandAndRecover(t *testing.T) { test := tests[i] t.Run(test.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, true) - client := csi.NewMockClient("foo", !test.disableNodeExpansion, !test.disableControllerExpansion, false, true, true, false) + client := csi.NewMockClient("foo", !test.disableNodeExpansion, !test.disableControllerExpansion, false, true, true) driverName, _ := client.GetDriverName(context.TODO()) if test.expansionError != nil { client.SetExpansionError(test.expansionError) diff --git a/pkg/controller/resize_status_test.go b/pkg/controller/resize_status_test.go index 7e13bfea6..67afc6844 100644 --- a/pkg/controller/resize_status_test.go +++ b/pkg/controller/resize_status_test.go @@ -77,7 +77,7 @@ func TestResizeFunctions(t *testing.T) { tc := test t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecoverVolumeExpansionFailure, true) - client := csi.NewMockClient("foo", true, true, false, true, true, false) + client := csi.NewMockClient("foo", true, true, false, true, true) driverName, _ := client.GetDriverName(context.TODO()) pvc := test.pvc diff --git a/pkg/csi/mock_client.go b/pkg/csi/mock_client.go index 49c93c3e2..dc8a03b7e 100644 --- a/pkg/csi/mock_client.go +++ b/pkg/csi/mock_client.go @@ -3,6 +3,8 @@ package csi import ( "context" "fmt" + "maps" + "sync" "sync/atomic" "github.com/container-storage-interface/spec/lib/go/csi" @@ -16,7 +18,6 @@ func NewMockClient( supportsControllerModify bool, supportsPluginControllerService bool, supportsControllerSingleNodeMultiWriter bool, - supportsExtraModifyMetada bool, ) *MockClient { return &MockClient{ name: name, @@ -25,7 +26,7 @@ func NewMockClient( supportsControllerModify: supportsControllerModify, supportsPluginControllerService: supportsPluginControllerService, supportsControllerSingleNodeMultiWriter: supportsControllerSingleNodeMultiWriter, - extraModifyMetadata: supportsExtraModifyMetada, + modifiedParameters: make(map[string]string), } } @@ -43,7 +44,8 @@ type MockClient struct { checkMigratedLabel bool usedSecrets atomic.Pointer[map[string]string] usedCapability atomic.Pointer[csi.VolumeCapability] - extraModifyMetadata bool + modifyMu sync.Mutex + modifiedParameters map[string]string } func (c *MockClient) GetDriverName(context.Context) (string, error) { @@ -116,6 +118,12 @@ func (c *MockClient) GetModifyCount() int { return int(c.modifyCalled.Load()) } +func (c *MockClient) GetModifiedParameters() map[string]string { + c.modifyMu.Lock() + defer c.modifyMu.Unlock() + return maps.Clone(c.modifiedParameters) +} + func (c *MockClient) GetCapability() *csi.VolumeCapability { return c.usedCapability.Load() } @@ -138,5 +146,8 @@ func (c *MockClient) Modify( if c.modifyError != nil { return c.modifyError } + c.modifyMu.Lock() + defer c.modifyMu.Unlock() + maps.Copy(c.modifiedParameters, mutableParameters) return nil } diff --git a/pkg/modifier/csi_modifier_test.go b/pkg/modifier/csi_modifier_test.go index de68df9f2..179eb0341 100644 --- a/pkg/modifier/csi_modifier_test.go +++ b/pkg/modifier/csi_modifier_test.go @@ -28,7 +28,7 @@ func TestNewModifier(t *testing.T) { SupportsControllerModify: false, }, } { - client := csi.NewMockClient("mock", false, false, c.SupportsControllerModify, false, false, false) + client := csi.NewMockClient("mock", false, false, c.SupportsControllerModify, false, false) driverName := "mock-driver" k8sClient, informerFactory := fakeK8s() _, err := NewModifierFromClient(client, 0, k8sClient, informerFactory, false, driverName) diff --git a/pkg/modifycontroller/controller.go b/pkg/modifycontroller/controller.go index 1895a3553..35185babb 100644 --- a/pkg/modifycontroller/controller.go +++ b/pkg/modifycontroller/controller.go @@ -19,6 +19,7 @@ package modifycontroller import ( "context" "fmt" + "sync" "time" "github.com/kubernetes-csi/external-resizer/pkg/util" @@ -38,6 +39,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog/v2" + "k8s.io/utils/ptr" ) // ModifyController watches PVCs and checks if they are requesting an modify operation. @@ -61,7 +63,7 @@ type modifyController struct { vacListerSynced cache.InformerSynced extraModifyMetadata bool // the key of the map is {PVC_NAMESPACE}/{PVC_NAME} - uncertainPVCs map[string]v1.PersistentVolumeClaim + uncertainPVCs sync.Map // slowSet tracks PVCs for which modification failed with infeasible error and should be retried at slower rate. slowSet *slowset.SlowSet } @@ -121,7 +123,6 @@ func NewModifyController( } func (ctrl *modifyController) initUncertainPVCs() error { - ctrl.uncertainPVCs = make(map[string]v1.PersistentVolumeClaim) allPVCs, err := ctrl.pvcLister.List(labels.Everything()) if err != nil { klog.Errorf("Failed to list pvcs when init uncertain pvcs: %v", err) @@ -133,7 +134,7 @@ func (ctrl *modifyController) initUncertainPVCs() error { if err != nil { return err } - ctrl.uncertainPVCs[pvcKey] = *pvc.DeepCopy() + ctrl.uncertainPVCs.Store(pvcKey, pvc) } } @@ -160,12 +161,11 @@ func (ctrl *modifyController) updatePVC(oldObj, newObj interface{}) { } // Only trigger modify volume if the following conditions are met - // 1. Non empty vac name - // 2. oldVacName != newVacName - // 3. PVC is in Bound state - oldVacName := oldPVC.Spec.VolumeAttributesClassName - newVacName := newPVC.Spec.VolumeAttributesClassName - if newVacName != nil && *newVacName != "" && (oldVacName == nil || *newVacName != *oldVacName) && oldPVC.Status.Phase == v1.ClaimBound { + // 1. oldVacName != newVacName + // 2. PVC is in Bound state + oldVacName := ptr.Deref(oldPVC.Spec.VolumeAttributesClassName, "") + newVacName := ptr.Deref(newPVC.Spec.VolumeAttributesClassName, "") + if newVacName != oldVacName && oldPVC.Status.Phase == v1.ClaimBound { _, err := ctrl.pvLister.Get(oldPVC.Spec.VolumeName) if err != nil { klog.Errorf("Get PV %q of pvc %q in PVInformer cache failed: %v", oldPVC.Spec.VolumeName, klog.KObj(oldPVC), err) @@ -187,10 +187,7 @@ func (ctrl *modifyController) deletePVC(obj interface{}) { } func (ctrl *modifyController) init(ctx context.Context) bool { - informersSyncd := []cache.InformerSynced{ctrl.pvListerSynced, ctrl.pvcListerSynced} - informersSyncd = append(informersSyncd, ctrl.vacListerSynced) - - if !cache.WaitForCacheSync(ctx.Done(), informersSyncd...) { + if !cache.WaitForCacheSync(ctx.Done(), ctrl.pvListerSynced, ctrl.pvcListerSynced, ctrl.vacListerSynced) { klog.ErrorS(nil, "Cannot sync pod, pv, pvc or vac caches") return false } @@ -220,7 +217,7 @@ func (ctrl *modifyController) Run( // Starts go-routine that deletes expired slowSet entries. go ctrl.slowSet.Run(stopCh) - for i := 0; i < workers; i++ { + for range workers { go wait.Until(ctrl.sync, 0, stopCh) } @@ -277,8 +274,7 @@ func (ctrl *modifyController) syncPVC(key string) error { return nil } - vacName := pvc.Spec.VolumeAttributesClassName - if vacName != nil && *vacName != "" && pvc.Status.Phase == v1.ClaimBound { + if pvc.Status.Phase == v1.ClaimBound { _, _, err, _ := ctrl.modify(pvc, pv) if err != nil { return err diff --git a/pkg/modifycontroller/controller_test.go b/pkg/modifycontroller/controller_test.go index 9ed3e077c..06e8fa529 100644 --- a/pkg/modifycontroller/controller_test.go +++ b/pkg/modifycontroller/controller_test.go @@ -1,39 +1,34 @@ package modifycontroller import ( - "context" "errors" "fmt" - "github.com/kubernetes-csi/external-resizer/pkg/util" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" "testing" "time" - "github.com/kubernetes-csi/external-resizer/pkg/features" - - "k8s.io/client-go/util/workqueue" - "github.com/kubernetes-csi/external-resizer/pkg/csi" + "github.com/kubernetes-csi/external-resizer/pkg/features" "github.com/kubernetes-csi/external-resizer/pkg/modifier" - + "github.com/kubernetes-csi/external-resizer/pkg/util" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" featuregatetesting "k8s.io/component-base/featuregate/testing" ) func TestController(t *testing.T) { basePVC := createTestPVC(pvcName, testVac /*vacName*/, testVac /*curVacName*/, testVac /*targetVacName*/) + basePVC.Status.ModifyVolumeStatus = nil basePV := createTestPV(1, pvcName, pvcNamespace, "foobaz" /*pvcUID*/, &fsVolumeMode, testVac) firstTimePV := basePV.DeepCopy() firstTimePV.Spec.VolumeAttributesClassName = nil firstTimePVC := basePVC.DeepCopy() firstTimePVC.Status.CurrentVolumeAttributesClassName = nil - firstTimePVC.Status.ModifyVolumeStatus = nil tests := []struct { name string @@ -68,11 +63,10 @@ func TestController(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Setup - client := csi.NewMockClient(testDriverName, true, true, true, true, true, false) + client := csi.NewMockClient(testDriverName, true, true, true, true, true) initialObjects := []runtime.Object{test.pvc, test.pv, testVacObject, targetVacObject} - ctrlInstance, ctx := setupFakeK8sEnvironment(t, client, initialObjects) - defer ctx.Done() + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) _, _, err, _ := ctrlInstance.modify(test.pvc, test.pv) if err != nil { @@ -120,14 +114,13 @@ func TestModifyPVC(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - client := csi.NewMockClient(testDriverName, true, true, true, true, true, false) + client := csi.NewMockClient(testDriverName, true, true, true, true, true) if test.modifyFailure { client.SetModifyError(fmt.Errorf("fake modification error")) } initialObjects := []runtime.Object{test.pvc, test.pv, testVacObject, targetVacObject} - ctrlInstance, ctx := setupFakeK8sEnvironment(t, client, initialObjects) - defer ctx.Done() + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) _, _, err, _ := ctrlInstance.modify(test.pvc, test.pv) @@ -188,6 +181,12 @@ func TestSyncPVC(t *testing.T) { pv: basePV, callCSIModify: true, }, + { + name: "Should execute ModifyVolume operation when rollback to empty VACName", + pvc: createTestPVC(pvcName, "" /*vacName*/, "" /*curVacName*/, testVac /*targetVacName*/), + pv: basePV, + callCSIModify: true, + }, { name: "Should NOT modify if PVC managed by another CSI Driver", pvc: basePVC, @@ -222,11 +221,10 @@ func TestSyncPVC(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - client := csi.NewMockClient(testDriverName, true, true, true, true, true, false) + client := csi.NewMockClient(testDriverName, true, true, true, true, true) initialObjects := []runtime.Object{test.pvc, test.pv, testVacObject, targetVacObject} - ctrlInstance, ctx := setupFakeK8sEnvironment(t, client, initialObjects) - defer ctx.Done() + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) err := ctrlInstance.syncPVC(pvcNamespace + "/" + pvcName) if err != nil { @@ -283,14 +281,13 @@ func TestInfeasibleRetry(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { // Setup - client := csi.NewMockClient(testDriverName, true, true, true, true, true, false) + client := csi.NewMockClient(testDriverName, true, true, true, true, true) if test.csiModifyError != nil { client.SetModifyError(test.csiModifyError) } initialObjects := []runtime.Object{test.pvc, basePV, testVacObject, targetVacObject} - ctrlInstance, ctx := setupFakeK8sEnvironment(t, client, initialObjects) - defer ctx.Done() + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) // Attempt modification first time err := ctrlInstance.syncPVC(pvcNamespace + "/" + pvcName) @@ -328,19 +325,76 @@ func TestInfeasibleRetry(t *testing.T) { } } +// Intended to catch any race conditions in the controller +func TestConcurrentSync(t *testing.T) { + cases := []struct { + name string + waitCount int + err error + }{ + // TODO: This case is flaky due to fake client lacks resourceVersion support. + // { + // name: "success", + // waitCount: 10, + // }, + { + name: "uncertain", + waitCount: 30, + err: nonFinalErr, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + client := csi.NewMockClient(testDriverName, true, true, true, true, true) + client.SetModifyError(tc.err) + + initialObjects := []runtime.Object{testVacObject, targetVacObject} + for i := range 10 { + initialObjects = append(initialObjects, + &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("foo-%d", i), Namespace: pvcNamespace}, + Spec: v1.PersistentVolumeClaimSpec{ + VolumeAttributesClassName: &testVac, + VolumeName: fmt.Sprintf("testPV-%d", i), + }, + Status: v1.PersistentVolumeClaimStatus{ + Phase: v1.ClaimBound, + }, + }, + &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("testPV-%d", i)}, + Spec: v1.PersistentVolumeSpec{ + PersistentVolumeSource: v1.PersistentVolumeSource{ + CSI: &v1.CSIPersistentVolumeSource{ + Driver: testDriverName, + VolumeHandle: fmt.Sprintf("foo-%d", i), + }, + }, + }, + }, + ) + } + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) + go ctrlInstance.Run(3, t.Context()) + + for client.GetModifyCount() < tc.waitCount { + time.Sleep(20 * time.Millisecond) + } + }) + } +} + // setupFakeK8sEnvironment creates fake K8s environment and starts Informers and ModifyController -func setupFakeK8sEnvironment(t *testing.T, client *csi.MockClient, initialObjects []runtime.Object) (*modifyController, context.Context) { +func setupFakeK8sEnvironment(t *testing.T, client *csi.MockClient, initialObjects []runtime.Object) *modifyController { t.Helper() featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) /* Create fake kubeClient, Informers, and ModifyController */ kubeClient, informerFactory := fakeK8s(initialObjects) - pvInformer := informerFactory.Core().V1().PersistentVolumes() - pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims() - vacInformer := informerFactory.Storage().V1beta1().VolumeAttributesClasses() - driverName, _ := client.GetDriverName(context.TODO()) + ctx := t.Context() + driverName, _ := client.GetDriverName(ctx) csiModifier, err := modifier.NewModifierFromClient(client, 15*time.Second, kubeClient, informerFactory, false, driverName) if err != nil { @@ -353,27 +407,10 @@ func setupFakeK8sEnvironment(t *testing.T, client *csi.MockClient, initialObject workqueue.DefaultTypedControllerRateLimiter[string]()) /* Start informers and ModifyController*/ - stopCh := make(chan struct{}) - informerFactory.Start(stopCh) - - ctx := context.TODO() - go controller.Run(1, ctx) - - /* Add initial objects to informer caches */ - for _, obj := range initialObjects { - switch obj.(type) { - case *v1.PersistentVolume: - pvInformer.Informer().GetStore().Add(obj) - case *v1.PersistentVolumeClaim: - pvcInformer.Informer().GetStore().Add(obj) - case *storagev1beta1.VolumeAttributesClass: - vacInformer.Informer().GetStore().Add(obj) - default: - t.Fatalf("Test %s: Unknown initalObject type: %+v", t.Name(), obj) - } - } + informerFactory.Start(ctx.Done()) - ctrlInstance, _ := controller.(*modifyController) + ctrl := controller.(*modifyController) + ctrl.init(ctx) - return ctrlInstance, ctx + return controller.(*modifyController) } diff --git a/pkg/modifycontroller/modify_status.go b/pkg/modifycontroller/modify_status.go index 4a12d034a..5cef18756 100644 --- a/pkg/modifycontroller/modify_status.go +++ b/pkg/modifycontroller/modify_status.go @@ -18,10 +18,11 @@ package modifycontroller import ( "fmt" + "github.com/kubernetes-csi/external-resizer/pkg/util" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/tools/cache" + "k8s.io/utils/ptr" ) // markControllerModifyVolumeStatus will mark ModifyVolumeStatus other than completed in the PVC @@ -31,10 +32,10 @@ func (ctrl *modifyController) markControllerModifyVolumeStatus( err error) (*v1.PersistentVolumeClaim, error) { newPVC := pvc.DeepCopy() - if newPVC.Status.ModifyVolumeStatus == nil { - newPVC.Status.ModifyVolumeStatus = &v1.ModifyVolumeStatus{} + newPVC.Status.ModifyVolumeStatus = &v1.ModifyVolumeStatus{ + Status: modifyVolumeStatus, + TargetVolumeAttributesClassName: ptr.Deref(pvc.Spec.VolumeAttributesClassName, ""), } - newPVC.Status.ModifyVolumeStatus.Status = modifyVolumeStatus // Update PVC's Condition to indicate modification pvcCondition := v1.PersistentVolumeClaimCondition{ Type: v1.PersistentVolumeClaimVolumeModifyingVolume, @@ -45,7 +46,6 @@ func (ctrl *modifyController) markControllerModifyVolumeStatus( switch modifyVolumeStatus { case v1.PersistentVolumeClaimModifyVolumeInProgress: conditionMessage = "ModifyVolume operation in progress." - newPVC.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName = *pvc.Spec.VolumeAttributesClassName case v1.PersistentVolumeClaimModifyVolumeInfeasible: conditionMessage = "ModifyVolume failed with error" + err.Error() + ". Waiting for retry." } @@ -60,16 +60,6 @@ func (ctrl *modifyController) markControllerModifyVolumeStatus( if err != nil { return pvc, fmt.Errorf("mark PVC %q as modify volume failed, errored with: %v", pvc.Name, err) } - // Remove this PVC from the uncertain cache since the status is known now - if modifyVolumeStatus == v1.PersistentVolumeClaimModifyVolumeInfeasible { - pvcKey, err := cache.MetaNamespaceKeyFunc(pvc) - if err != nil { - return pvc, err - } - - ctrl.removePVCFromModifyVolumeUncertainCache(pvcKey) - ctrl.markForSlowRetry(pvc, pvcKey) - } return updatedPVC, nil } @@ -99,7 +89,10 @@ func (ctrl *modifyController) updateConditionBasedOnError(pvc *v1.PersistentVolu // markControllerModifyVolumeStatus will mark ModifyVolumeStatus as completed in the PVC // and update CurrentVolumeAttributesClassName, clear the conditions func (ctrl *modifyController) markControllerModifyVolumeCompleted(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error) { - modifiedVacName := pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName + var modifiedVacName *string + if pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName != "" { + modifiedVacName = &pvc.Status.ModifyVolumeStatus.TargetVolumeAttributesClassName + } // Update PVC newPVC := pvc.DeepCopy() @@ -108,14 +101,14 @@ func (ctrl *modifyController) markControllerModifyVolumeCompleted(pvc *v1.Persis newPVC.Status.ModifyVolumeStatus = nil // Update CurrentVolumeAttributesClassName - newPVC.Status.CurrentVolumeAttributesClassName = &modifiedVacName + newPVC.Status.CurrentVolumeAttributesClassName = modifiedVacName // Clear all the conditions related to modify volume newPVC.Status.Conditions = clearModifyVolumeConditions(newPVC.Status.Conditions) // Update PV newPV := pv.DeepCopy() - newPV.Spec.VolumeAttributesClassName = &modifiedVacName + newPV.Spec.VolumeAttributesClassName = modifiedVacName // Update PV before PVC to avoid PV not getting updated but PVC did updatedPV, err := util.PatchPersistentVolume(ctrl.kubeClient, pv, newPV) @@ -143,15 +136,3 @@ func clearModifyVolumeConditions(conditions []v1.PersistentVolumeClaimCondition) } return knownConditions } - -// removePVCFromModifyVolumeUncertainCache removes the pvc from the uncertain cache -func (ctrl *modifyController) removePVCFromModifyVolumeUncertainCache(pvcKey string) { - if ctrl.uncertainPVCs == nil { - return - } - // Format of the key of the uncertainPVCs is NAMESPACE/NAME of the pvc - _, ok := ctrl.uncertainPVCs[pvcKey] - if ok { - delete(ctrl.uncertainPVCs, pvcKey) - } -} diff --git a/pkg/modifycontroller/modify_status_test.go b/pkg/modifycontroller/modify_status_test.go index 932245a65..a7b420ae1 100644 --- a/pkg/modifycontroller/modify_status_test.go +++ b/pkg/modifycontroller/modify_status_test.go @@ -14,13 +14,11 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" - "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" featuregatetesting "k8s.io/component-base/featuregate/testing" ) @@ -38,6 +36,7 @@ var ( testDriverName = "mock" infeasibleErr = status.Errorf(codes.InvalidArgument, "Parameters in VolumeAttributesClass is invalid") finalErr = status.Errorf(codes.Internal, "Final error") + nonFinalErr = status.Errorf(codes.Aborted, "Non-final error") pvcConditionInProgress = v1.PersistentVolumeClaimCondition{ Type: v1.PersistentVolumeClaimVolumeModifyingVolume, Status: v1.ConditionTrue, @@ -104,7 +103,7 @@ func TestMarkControllerModifyVolumeStatus(t *testing.T) { tc := test t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) - client := csi.NewMockClient("foo", true, true, true, true, true, false) + client := csi.NewMockClient("foo", true, true, true, true, true) driverName, _ := client.GetDriverName(context.TODO()) pvc := test.pvc @@ -164,7 +163,7 @@ func TestUpdateConditionBasedOnError(t *testing.T) { tc := test t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) - client := csi.NewMockClient("foo", true, true, true, true, true, false) + client := csi.NewMockClient("foo", true, true, true, true, true) driverName, _ := client.GetDriverName(context.TODO()) pvc := test.pvc @@ -233,7 +232,7 @@ func TestMarkControllerModifyVolumeCompleted(t *testing.T) { tc := test t.Run(tc.name, func(t *testing.T) { featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) - client := csi.NewMockClient("foo", true, true, true, true, true, false) + client := csi.NewMockClient("foo", true, true, true, true, true) driverName, _ := client.GetDriverName(context.TODO()) var initialObjects []runtime.Object @@ -273,112 +272,6 @@ func TestMarkControllerModifyVolumeCompleted(t *testing.T) { } } -func TestRemovePVCFromModifyVolumeUncertainCache(t *testing.T) { - basePVC := testutil.MakeTestPVC([]v1.PersistentVolumeClaimCondition{}) - basePVC.WithModifyVolumeStatus(v1.PersistentVolumeClaimModifyVolumeInProgress) - secondPVC := testutil.GetTestPVC("test-vol0", "2G", "1G", "", "") - secondPVC.Status.Phase = v1.ClaimBound - secondPVC.Status.ModifyVolumeStatus = &v1.ModifyVolumeStatus{} - secondPVC.Status.ModifyVolumeStatus.Status = v1.PersistentVolumeClaimModifyVolumeInfeasible - - tests := []struct { - name string - pvc *v1.PersistentVolumeClaim - }{ - { - name: "should delete the target pvc but keep the others in the cache", - pvc: basePVC.Get(), - }, - } - - for _, test := range tests { - tc := test - t.Run(tc.name, func(t *testing.T) { - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.VolumeAttributesClass, true) - client := csi.NewMockClient("foo", true, true, true, true, true, false) - driverName, _ := client.GetDriverName(context.TODO()) - - var initialObjects []runtime.Object - initialObjects = append(initialObjects, test.pvc) - initialObjects = append(initialObjects, secondPVC) - - kubeClient, informerFactory := fakeK8s(initialObjects) - pvInformer := informerFactory.Core().V1().PersistentVolumes() - pvcInformer := informerFactory.Core().V1().PersistentVolumeClaims() - podInformer := informerFactory.Core().V1().Pods() - vacInformer := informerFactory.Storage().V1beta1().VolumeAttributesClasses() - - csiModifier, err := modifier.NewModifierFromClient(client, 15*time.Second, kubeClient, informerFactory, false, driverName) - if err != nil { - t.Fatalf("Test %s: Unable to create modifier: %v", test.name, err) - } - controller := NewModifyController(driverName, - csiModifier, kubeClient, - time.Second, 2*time.Minute, false, informerFactory, - workqueue.DefaultTypedControllerRateLimiter[string]()) - - ctrlInstance, _ := controller.(*modifyController) - - stopCh := make(chan struct{}) - informerFactory.Start(stopCh) - - ctx := context.TODO() - defer ctx.Done() - success := ctrlInstance.init(ctx) - if !success { - t.Fatal("failed to init controller") - } - - for _, obj := range initialObjects { - switch obj.(type) { - case *v1.PersistentVolume: - pvInformer.Informer().GetStore().Add(obj) - case *v1.PersistentVolumeClaim: - pvcInformer.Informer().GetStore().Add(obj) - case *v1.Pod: - podInformer.Informer().GetStore().Add(obj) - case *storagev1beta1.VolumeAttributesClass: - vacInformer.Informer().GetStore().Add(obj) - default: - t.Fatalf("Test %s: Unknown initalObject type: %+v", test.name, obj) - } - } - - time.Sleep(time.Second * 2) - - pvcKey, err := cache.MetaNamespaceKeyFunc(tc.pvc) - if err != nil { - t.Errorf("failed to extract pvc key from pvc %v", tc.pvc) - } - ctrlInstance.removePVCFromModifyVolumeUncertainCache(pvcKey) - - deletedPVCKey, err := cache.MetaNamespaceKeyFunc(tc.pvc) - if err != nil { - t.Errorf("failed to extract pvc key from pvc %v", tc.pvc) - } - _, ok := ctrlInstance.uncertainPVCs[deletedPVCKey] - if ok { - t.Errorf("pvc %v should be deleted but it is still in the uncertainPVCs cache", tc.pvc) - } - if err != nil { - t.Errorf("err get pvc %v from uncertainPVCs: %v", tc.pvc, err) - } - - notDeletedPVCKey, err := cache.MetaNamespaceKeyFunc(secondPVC) - if err != nil { - t.Errorf("failed to extract pvc key from secondPVC %v", secondPVC) - } - _, ok = ctrlInstance.uncertainPVCs[notDeletedPVCKey] - if !ok { - t.Errorf("pvc %v should not be deleted, uncertainPVCs list %v", secondPVC, ctrlInstance.uncertainPVCs) - } - if err != nil { - t.Errorf("err get pvc %v from uncertainPVCs: %v", secondPVC, err) - } - }) - } -} - func createTestPV(capacityGB int, pvcName, pvcNamespace string, pvcUID types.UID, volumeMode *v1.PersistentVolumeMode, vacName string) *v1.PersistentVolume { capacity := testutil.QuantityGB(capacityGB) diff --git a/pkg/modifycontroller/modify_volume.go b/pkg/modifycontroller/modify_volume.go index 7523ab624..130486b14 100644 --- a/pkg/modifycontroller/modify_volume.go +++ b/pkg/modifycontroller/modify_volume.go @@ -18,6 +18,7 @@ package modifycontroller import ( "fmt" + "maps" "time" "github.com/kubernetes-csi/csi-lib-utils/slowset" @@ -25,10 +26,10 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" v1 "k8s.io/api/core/v1" - storagev1beta1 "k8s.io/api/storage/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + "k8s.io/utils/ptr" ) const ( @@ -39,8 +40,6 @@ const ( // The return value bool is only used as a sentinel value when function returns without actually performing modification func (ctrl *modifyController) modify(pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error, bool) { - pvcSpecVacName := pvc.Spec.VolumeAttributesClassName - curVacName := pvc.Status.CurrentVolumeAttributesClassName pvcKey, err := cache.MetaNamespaceKeyFunc(pvc) if err != nil { return pvc, pv, err, false @@ -52,29 +51,48 @@ func (ctrl *modifyController) modify(pvc *v1.PersistentVolumeClaim, pv *v1.Persi return pvc, pv, delayModificationErr, false } - if pvcSpecVacName != nil && curVacName == nil { - // First time adding VAC to a PVC - return ctrl.validateVACAndModifyVolumeWithTarget(pvc, pv) - } else if pvcSpecVacName != nil && curVacName != nil && *pvcSpecVacName != *curVacName { - // Check if PVC in uncertain state - _, inUncertainState := ctrl.uncertainPVCs[pvcKey] - if !inUncertainState { - klog.V(3).InfoS("previous operation on the PVC failed with a final error, retrying") - return ctrl.validateVACAndModifyVolumeWithTarget(pvc, pv) - } else { - vac, err := ctrl.vacLister.Get(*pvcSpecVacName) - if err != nil { - if apierrors.IsNotFound(err) { - ctrl.eventRecorder.Eventf(pvc, v1.EventTypeWarning, util.VolumeModifyFailed, "VAC "+*pvcSpecVacName+" does not exist.") - } - return pvc, pv, err, false - } - return ctrl.controllerModifyVolumeWithTarget(pvc, pv, vac, pvcSpecVacName) + pvcSpecVacName := ptr.Deref(pvc.Spec.VolumeAttributesClassName, "") + curVacName := ptr.Deref(pvc.Status.CurrentVolumeAttributesClassName, "") + if pvc.Status.ModifyVolumeStatus == nil && pvcSpecVacName == curVacName { + // No modification required + return pvc, pv, nil, false + } + + if pvcSpecVacName == "" && curVacName != "" { + klog.V(4).InfoS("Can only set VAC to empty for rollback", "PV", klog.KObj(pv)) + return pvc, pv, nil, false + } + + // Check if PVC in uncertain state + _, inUncertainState := ctrl.uncertainPVCs.Load(pvcKey) + if inUncertainState { + pvcSpecVacName, parameters, err := ctrl.getTargetParameters(pvc) + if err != nil { + return pvc, pv, err, false } + return ctrl.controllerModifyVolumeWithTarget(pvc, pv, parameters, pvcSpecVacName) } - // No modification required - return pvc, pv, nil, false + return ctrl.validateVACAndModifyVolumeWithTarget(pvc, pv) +} + +func (ctrl *modifyController) getTargetParameters(pvc *v1.PersistentVolumeClaim) (pvcSpecVacName string, parameters map[string]string, err error) { + if pvc.Spec.VolumeAttributesClassName == nil || *pvc.Spec.VolumeAttributesClassName == "" { + pvcSpecVacName = "[nil]" + } else { + pvcSpecVacName = *pvc.Spec.VolumeAttributesClassName + vac, err := ctrl.vacLister.Get(pvcSpecVacName) + // Check if pvcSpecVac is valid and exist + if err != nil { + if apierrors.IsNotFound(err) { + ctrl.eventRecorder.Eventf(pvc, v1.EventTypeWarning, util.VolumeModifyFailed, "VAC "+pvcSpecVacName+" does not exist.") + } + klog.Errorf("Get VAC with vac name %s in VACInformer cache failed: %v", pvcSpecVacName, err) + return "", nil, err + } + parameters = vac.Parameters + } + return pvcSpecVacName, parameters, nil } // func validateVACAndModifyVolumeWithTarget validate the VAC. The function sets pvc.Status.ModifyVolumeStatus @@ -82,29 +100,25 @@ func (ctrl *modifyController) modify(pvc *v1.PersistentVolumeClaim, pv *v1.Persi func (ctrl *modifyController) validateVACAndModifyVolumeWithTarget( pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error, bool) { - // The controller only triggers ModifyVolume if pvcSpecVacName is not nil nor empty - pvcSpecVacName := pvc.Spec.VolumeAttributesClassName - // Check if pvcSpecVac is valid and exist - vac, err := ctrl.vacLister.Get(*pvcSpecVacName) - if err == nil { - // Mark pvc.Status.ModifyVolumeStatus as in progress - pvc, err = ctrl.markControllerModifyVolumeStatus(pvc, v1.PersistentVolumeClaimModifyVolumeInProgress, nil) - if err != nil { - return pvc, pv, err, false - } - // Record an event to indicate that external resizer is modifying this volume. - ctrl.eventRecorder.Event(pvc, v1.EventTypeNormal, util.VolumeModify, - fmt.Sprintf("external resizer is modifying volume %s with vac %s", pvc.Name, *pvcSpecVacName)) - return ctrl.controllerModifyVolumeWithTarget(pvc, pv, vac, pvcSpecVacName) - } else { + + pvcSpecVacName, parameters, err := ctrl.getTargetParameters(pvc) + if err != nil { if apierrors.IsNotFound(err) { - ctrl.eventRecorder.Eventf(pvc, v1.EventTypeWarning, util.VolumeModifyFailed, "VAC "+*pvcSpecVacName+" does not exist.") + // Mark pvc.Status.ModifyVolumeStatus as pending + pvc, err = ctrl.markControllerModifyVolumeStatus(pvc, v1.PersistentVolumeClaimModifyVolumePending, nil) } - klog.Errorf("Get VAC with vac name %s in VACInformer cache failed: %v", *pvcSpecVacName, err) - // Mark pvc.Status.ModifyVolumeStatus as pending - pvc, err = ctrl.markControllerModifyVolumeStatus(pvc, v1.PersistentVolumeClaimModifyVolumePending, nil) return pvc, pv, err, false } + + // Mark pvc.Status.ModifyVolumeStatus as in progress + pvc, err = ctrl.markControllerModifyVolumeStatus(pvc, v1.PersistentVolumeClaimModifyVolumeInProgress, nil) + if err != nil { + return pvc, pv, err, false + } + // Record an event to indicate that external resizer is modifying this volume. + ctrl.eventRecorder.Event(pvc, v1.EventTypeNormal, util.VolumeModify, + fmt.Sprintf("external resizer is modifying volume %s with vac %s", pvc.Name, pvcSpecVacName)) + return ctrl.controllerModifyVolumeWithTarget(pvc, pv, parameters, pvcSpecVacName) } // func controllerModifyVolumeWithTarget trigger the CSI ControllerModifyVolume API call @@ -112,14 +126,14 @@ func (ctrl *modifyController) validateVACAndModifyVolumeWithTarget( func (ctrl *modifyController) controllerModifyVolumeWithTarget( pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, - vacObj *storagev1beta1.VolumeAttributesClass, - pvcSpecVacName *string) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error, bool) { + parameters map[string]string, + pvcSpecVacName string) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error, bool) { var err error - pvc, pv, err = ctrl.callModifyVolumeOnPlugin(pvc, pv, vacObj) + pvc, pv, err = ctrl.callModifyVolumeOnPlugin(pvc, pv, parameters) if err == nil { - klog.V(4).Infof("Update volumeAttributesClass of PV %q to %s succeeded", pv.Name, *pvcSpecVacName) + klog.V(4).Infof("Update volumeAttributesClass of PV %q to %s succeeded", pv.Name, pvcSpecVacName) // Record an event to indicate that modify operation is successful. - ctrl.eventRecorder.Eventf(pvc, v1.EventTypeNormal, util.VolumeModifySuccess, fmt.Sprintf("external resizer modified volume %s with vac %s successfully ", pvc.Name, vacObj.Name)) + ctrl.eventRecorder.Eventf(pvc, v1.EventTypeNormal, util.VolumeModifySuccess, fmt.Sprintf("external resizer modified volume %s with vac %s successfully", pvc.Name, pvcSpecVacName)) return pvc, pv, nil, true } else { errStatus, ok := status.FromError(err) @@ -132,9 +146,9 @@ func (ctrl *modifyController) controllerModifyVolumeWithTarget( if keyErr != nil { return pvc, pv, keyErr, false } - if !util.IsFinalError(keyErr) { + if !util.IsFinalError(err) { // update conditions and cache pvc as uncertain - ctrl.uncertainPVCs[pvcKey] = *pvc + ctrl.uncertainPVCs.Store(pvcKey, pvc) } else { // Only InvalidArgument can be set to Infeasible state // Final errors other than InvalidArgument will still be in InProgress state @@ -146,10 +160,10 @@ func (ctrl *modifyController) controllerModifyVolumeWithTarget( } ctrl.markForSlowRetry(pvc, pvcKey) } - ctrl.removePVCFromModifyVolumeUncertainCache(pvcKey) + ctrl.uncertainPVCs.Delete(pvcKey) } } else { - return pvc, pv, fmt.Errorf("cannot get error status from modify volume err: %v ", err), false + return pvc, pv, fmt.Errorf("cannot get error status from modify volume err: %v", err), false } // Record an event to indicate that modify operation is failed. ctrl.eventRecorder.Eventf(pvc, v1.EventTypeWarning, util.VolumeModifyFailed, err.Error()) @@ -160,13 +174,18 @@ func (ctrl *modifyController) controllerModifyVolumeWithTarget( func (ctrl *modifyController) callModifyVolumeOnPlugin( pvc *v1.PersistentVolumeClaim, pv *v1.PersistentVolume, - vac *storagev1beta1.VolumeAttributesClass) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error) { + parameters map[string]string) (*v1.PersistentVolumeClaim, *v1.PersistentVolume, error) { if ctrl.extraModifyMetadata { - vac.Parameters[pvcNameKey] = pvc.GetName() - vac.Parameters[pvcNamespaceKey] = pvc.GetNamespace() - vac.Parameters[pvNameKey] = pv.GetName() + if len(parameters) == 0 { + parameters = make(map[string]string, 3) + } else { + parameters = maps.Clone(parameters) + } + parameters[pvcNameKey] = pvc.GetName() + parameters[pvcNamespaceKey] = pvc.GetNamespace() + parameters[pvNameKey] = pv.GetName() } - err := ctrl.modifier.Modify(pv, vac.Parameters) + err := ctrl.modifier.Modify(pv, parameters) if err != nil { return pvc, pv, err diff --git a/pkg/modifycontroller/modify_volume_test.go b/pkg/modifycontroller/modify_volume_test.go index 25a8abbe8..aaad6d116 100644 --- a/pkg/modifycontroller/modify_volume_test.go +++ b/pkg/modifycontroller/modify_volume_test.go @@ -1,6 +1,8 @@ package modifycontroller import ( + "errors" + "fmt" "testing" "github.com/google/go-cmp/cmp" @@ -25,17 +27,13 @@ var ( targetVacObject = &storagev1beta1.VolumeAttributesClass{ ObjectMeta: metav1.ObjectMeta{Name: targetVac}, DriverName: testDriverName, - Parameters: map[string]string{ - "iops": "4567", - "csi.storage.k8s.io/pvc/name": pvcName, - "csi.storage.k8s.io/pvc/namespace": pvcNamespace, - "csi.storage.k8s.io/pv/name": pvName, - }, + Parameters: map[string]string{"iops": "4567"}, } ) func TestModify(t *testing.T) { basePVC := createTestPVC(pvcName, testVac /*vacName*/, testVac /*curVacName*/, testVac /*targetVacName*/) + basePVC.Status.ModifyVolumeStatus = nil basePV := createTestPV(1, pvcName, pvcNamespace, "foobaz" /*pvcUID*/, &fsVolumeMode, testVac) var tests = []struct { @@ -48,7 +46,7 @@ func TestModify(t *testing.T) { expectedCurrentVolumeAttributesClassName *string expectedPVVolumeAttributesClassName *string withExtraMetadata bool - expectedVacParams map[string]string + expectedMutableParams map[string]string }{ { name: "nothing to modify", @@ -61,11 +59,11 @@ func TestModify(t *testing.T) { }, { name: "vac does not exist, no modification and set ModifyVolumeStatus to pending", - pvc: createTestPVC(pvcName, targetVac /*vacName*/, testVac /*curVacName*/, testVac /*targetVacName*/), + pvc: createTestPVC(pvcName, targetVac /*vacName*/, testVac /*curVacName*/, "" /*targetVacName*/), pv: basePV, expectModifyCall: false, expectedModifyVolumeStatus: &v1.ModifyVolumeStatus{ - TargetVolumeAttributesClassName: testVac, + TargetVolumeAttributesClassName: targetVac, Status: v1.PersistentVolumeClaimModifyVolumePending, }, expectedCurrentVolumeAttributesClassName: &testVac, @@ -80,6 +78,7 @@ func TestModify(t *testing.T) { expectedModifyVolumeStatus: nil, expectedCurrentVolumeAttributesClassName: &targetVac, expectedPVVolumeAttributesClassName: &targetVac, + expectedMutableParams: map[string]string{"iops": "4567"}, }, { name: "modify volume success with extra metadata", @@ -91,7 +90,7 @@ func TestModify(t *testing.T) { expectedCurrentVolumeAttributesClassName: &targetVac, expectedPVVolumeAttributesClassName: &targetVac, withExtraMetadata: true, - expectedVacParams: map[string]string{ + expectedMutableParams: map[string]string{ "iops": "4567", "csi.storage.k8s.io/pvc/name": basePVC.GetName(), "csi.storage.k8s.io/pvc/namespace": basePVC.GetNamespace(), @@ -104,13 +103,13 @@ func TestModify(t *testing.T) { test := tests[i] t.Run(test.name, func(t *testing.T) { // Setup - client := csi.NewMockClient(testDriverName, true, true, true, true, true, test.withExtraMetadata) + client := csi.NewMockClient(testDriverName, true, true, true, true, true) initialObjects := []runtime.Object{test.pvc, test.pv, testVacObject} if test.vacExists { initialObjects = append(initialObjects, targetVacObject) } - ctrlInstance, ctx := setupFakeK8sEnvironment(t, client, initialObjects) - defer ctx.Done() + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) + ctrlInstance.extraModifyMetadata = test.withExtraMetadata // Action pvc, pv, err, modifyCalled := ctrlInstance.modify(test.pvc, test.pv) @@ -139,21 +138,54 @@ func TestModify(t *testing.T) { t.Errorf("expected VolumeAttributesClassName of pv to be %v, got %v", *test.expectedPVVolumeAttributesClassName, *actualPVVolumeAttributesClassName) } - if test.withExtraMetadata { - vacObj, err := ctrlInstance.vacLister.Get(*test.expectedPVVolumeAttributesClassName) - if err != nil { - t.Errorf("failed to get VAC: %v", err) - } else { - vacParams := vacObj.Parameters - if diff := cmp.Diff(test.expectedVacParams, vacParams); diff != "" { - t.Errorf("expected VAC parameters to be %v, got %v", test.expectedVacParams, vacParams) - } + if test.expectedMutableParams != nil { + p := client.GetModifiedParameters() + if diff := cmp.Diff(test.expectedMutableParams, p); diff != "" { + t.Errorf("expected mutable parameters to be %v, got %v", test.expectedMutableParams, p) } } }) } } +func TestModifyUncertain(t *testing.T) { + basePVC := createTestPVC(pvcName, testVac /*vacName*/, testVac /*curVacName*/, targetVac /*targetVacName*/) + basePVC.Status.ModifyVolumeStatus.Status = v1.PersistentVolumeClaimModifyVolumeInProgress + basePV := createTestPV(1, pvcName, pvcNamespace, "foobaz" /*pvcUID*/, &fsVolumeMode, testVac) + + client := csi.NewMockClient(testDriverName, true, true, true, true, true) + initialObjects := []runtime.Object{testVacObject, targetVacObject, basePVC, basePV} + ctrlInstance := setupFakeK8sEnvironment(t, client, initialObjects) + + pvcKey := fmt.Sprintf("%s/%s", pvcNamespace, pvcName) + assertUncertain := func(uncertain bool) { + t.Helper() + _, ok := ctrlInstance.uncertainPVCs.Load(pvcKey) + if ok != uncertain { + t.Fatalf("expected uncertain state to be %v, got %v", uncertain, ok) + } + } + + // initialized to uncertain + assertUncertain(true) + + client.SetModifyError(finalErr) + pvc, pv, err, _ := ctrlInstance.modify(basePVC, basePV) + if !errors.Is(err, finalErr) { + t.Fatalf("expected error to be %v, got %v", finalErr, err) + } + // should clear uncertain state + assertUncertain(false) + + client.SetModifyError(nonFinalErr) + _, _, err, _ = ctrlInstance.modify(pvc, pv) + if !errors.Is(err, nonFinalErr) { + t.Fatalf("expected error to be %v, got %v", nonFinalErr, err) + } + // should enter uncertain state again + assertUncertain(true) +} + func createTestPVC(pvcName string, vacName string, curVacName string, targetVacName string) *v1.PersistentVolumeClaim { pvc := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: pvcName, Namespace: pvcNamespace}, diff --git a/pkg/resizer/csi_resizer_test.go b/pkg/resizer/csi_resizer_test.go index 6d089511f..ea6b9e023 100644 --- a/pkg/resizer/csi_resizer_test.go +++ b/pkg/resizer/csi_resizer_test.go @@ -72,7 +72,7 @@ func TestNewResizer(t *testing.T) { Error: resizeNotSupportErr, }, } { - client := csi.NewMockClient("mock", c.SupportsNodeResize, c.SupportsControllerResize, false, c.SupportsPluginControllerService, c.SupportsControllerSingleNodeMultiWriter, false) + client := csi.NewMockClient("mock", c.SupportsNodeResize, c.SupportsControllerResize, false, c.SupportsPluginControllerService, c.SupportsControllerSingleNodeMultiWriter) driverName := "mock-driver" k8sClient := fake.NewSimpleClientset() resizer, err := NewResizerFromClient(client, 0, k8sClient, driverName) @@ -106,7 +106,7 @@ func TestResizeWithSecret(t *testing.T) { }, } for _, tc := range tests { - client := csi.NewMockClient("mock", true, true, false, true, true, false) + client := csi.NewMockClient("mock", true, true, false, true, true) secret := makeSecret("some-secret", "secret-namespace") k8sClient := fake.NewSimpleClientset(secret) pv := makeTestPV("test-csi", 2, "ebs-csi", "vol-abcde", tc.hasExpansionSecret) @@ -164,7 +164,7 @@ func TestResizeMigratedPV(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { driverName := tc.driverName - client := csi.NewMockClient(driverName, true, true, false, true, true, false) + client := csi.NewMockClient(driverName, true, true, false, true, true) client.SetCheckMigratedLabel() k8sClient := fake.NewSimpleClientset() resizer, err := NewResizerFromClient(client, 0, k8sClient, driverName) @@ -433,7 +433,7 @@ func TestCanSupport(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { driverName := tc.driverName - client := csi.NewMockClient(driverName, true, true, false, true, true, false) + client := csi.NewMockClient(driverName, true, true, false, true, true) k8sClient := fake.NewSimpleClientset() resizer, err := NewResizerFromClient(client, 0, k8sClient, driverName) if err != nil {