feat(data engine): Add the VolumeAttributesClass feature

cmd/main.go

@@ -134,6 +134,10 @@ func main() {
 		&config.NodeControllerPollingInterval, "node-polling-interval", 60, "The interval, in seconds, between node polling.",
 	)
 
+	cmd.PersistentFlags().StringVar(
+		&config.KubeletDir, "kubelet-dir", "/var/lib/kubelet/", "Kubelet directory provides the operation of volumeAttributesClassName",
+	)
+
 	err := cmd.Execute()
 	if err != nil {
 		_, _ = fmt.Fprintf(os.Stderr, "%s", err.Error())
@@ -165,6 +169,8 @@ func run(config *config.Config) {
 		lvm.SetIORateLimits(config)
 	}
 
+	lvm.SetQoSValuesConf(config)
+
 	err := driver.New(config).Run()
 	if err != nil {
 		log.Fatalln(err)

deploy/helm/charts/charts/crds/templates/lvmvolume.yaml

@@ -80,6 +80,25 @@ spec:
                   can not be edited after the volume has been provisioned.
                 minLength: 1
                 type: string
+              qos:
+                description: |-
+                  QoS contains VAC realization based on cgroup v2 io controller.
+                  Nil means "no explicit QoS settings applied".
+                properties:
+                  readBPS:
+                    type: string
+                  readIOPS:
+                    type: string
+                  writeBPS:
+                    type: string
+                  writeIOPS:
+                    type: string
+                required:
+                - readBPS
+                - readIOPS
+                - writeBPS
+                - writeIOPS
+                type: object
               shared:
                 description: Shared specifies whether the volume can be shared among
                   multiple pods. If it is not set to "yes", then the LVM LocalPV Driver

deploy/helm/charts/templates/lvm-node.yaml

@@ -85,6 +85,9 @@ spec:
             {{- if .Values.lvmPlugin.metricsPort }}
             - "--listen-address=$(METRICS_LISTEN_ADDRESS)"
             {{- end }}
+            {{- if .Values.lvmNode.kubeletDir }}
+            - "--kubelet-dir={{ include "lvmlocalpv.lvmNode.kubeletDir" . }}"
+            {{- end }}
           env:
             - name: OPENEBS_NODE_ID
               valueFrom:

deploy/helm/charts/templates/rbac.yaml

@@ -27,7 +27,7 @@ rules:
     resources: ["persistentvolumeclaims/status"]
     verbs: ["update", "patch"]
   - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses", "csinodes"]
+    resources: ["storageclasses", "csinodes","volumeattributesclasses"]
     verbs: ["get", "list", "watch"]
   - apiGroups: [ "storage.k8s.io" ]
     resources: [ "csistoragecapacities"]

deploy/helm/charts/values.yaml

@@ -85,7 +85,7 @@ lvmController:
       repository: sig-storage/csi-resizer
       pullPolicy: IfNotPresent
       # Overrides the image tag whose default is the chart appVersion.
-      tag: v1.11.2
+      tag: v2.0.0
   snapshotter:
     name: "csi-snapshotter"
     image:
@@ -115,7 +115,7 @@ lvmController:
       repository: sig-storage/csi-provisioner
       pullPolicy: IfNotPresent
       # Overrides the image tag whose default is the chart appVersion.
-      tag: v5.2.0
+      tag: v6.1.0
   updateStrategy:
     type: RollingUpdate
   annotations: {}

deploy/lvm-operator.yaml

@@ -105,6 +105,25 @@ spec:
                   OwnerNodeID can not be edited after the volume has been provisioned.
                 minLength: 1
                 type: string
+              qos:
+                description: |-
+                  QoS contains VAC realization based on cgroup v2 io controller.
+                  Nil means "no explicit QoS settings applied".
+                properties:
+                  readBPS:
+                    type: string
+                  readIOPS:
+                    type: string
+                  writeBPS:
+                    type: string
+                  writeIOPS:
+                    type: string
+                required:
+                - readBPS
+                - readIOPS
+                - writeBPS
+                - writeIOPS
+                type: object
               shared:
                 description: |-
                   Shared specifies whether the volume can be shared among multiple pods.

deploy/yamls/lvmvolume-crd.yaml

@@ -84,6 +84,25 @@ spec:
                   OwnerNodeID can not be edited after the volume has been provisioned.
                 minLength: 1
                 type: string
+              qos:
+                description: |-
+                  QoS contains VAC realization based on cgroup v2 io controller.
+                  Nil means "no explicit QoS settings applied".
+                properties:
+                  readBPS:
+                    type: string
+                  readIOPS:
+                    type: string
+                  writeBPS:
+                    type: string
+                  writeIOPS:
+                    type: string
+                required:
+                - readBPS
+                - readIOPS
+                - writeBPS
+                - writeIOPS
+                type: object
               shared:
                 description: |-
                   Shared specifies whether the volume can be shared among multiple pods.

pkg/apis/openebs.io/lvm/v1alpha1/lvmvolume.go

@@ -74,6 +74,10 @@ type VolumeInfo struct {
 	// +kubebuilder:validation:MinLength=1
 	Capacity string `json:"capacity"`
 
+	// QoS contains VAC realization based on cgroup v2 io controller.
+	// Nil means "no explicit QoS settings applied".
+	QoS *VolumeQoS `json:"qos,omitempty"`
+
 	// Shared specifies whether the volume can be shared among multiple pods.
 	// If it is not set to "yes", then the LVM LocalPV Driver will not allow
 	// the volumes to be mounted by more than one pods.
@@ -93,6 +97,15 @@ type VolumeInfo struct {
 	Source string `json:"source,omitempty"`
 }
 
+// VolumeQoS now represents per-volume IO management based on cgroup v2 io.max.
+// Each field be either "max" or a positive integer string (base-10).
+type VolumeQoS struct {
+	ReadBPS   string `json:"readBPS"`
+	ReadIOPS  string `json:"readIOPS"`
+	WriteBPS  string `json:"writeBPS"`
+	WriteIOPS string `json:"writeIOPS"`
+}
+
 // VolStatus string that specifies the current state of the volume provisioning request.
 type VolStatus struct {
 	// State specifies the current state of the volume provisioning request.

pkg/builder/volbuilder/volume.go

@@ -213,6 +213,18 @@ func (b *Builder) WithSource(source string) *Builder {
 	return b
 }
 
+// WithQoS sets optional per-volume distribution of IO resources
+func (b *Builder) WithQoS(qos *apis.VolumeQoS) *Builder {
+	if qos == nil {
+		b.volume.Object.Spec.QoS = nil
+		return b
+	}
+	cp := new(apis.VolumeQoS)
+	*cp = *qos
+	b.volume.Object.Spec.QoS = cp
+	return b
+}
+
 // Build returns LVMVolume API object
 func (b *Builder) Build() (*apis.LVMVolume, error) {
 	if len(b.errs) > 0 {

pkg/driver/config/config.go

@@ -88,6 +88,9 @@ type Config struct {
 
 	// NodeControllerPollingInterval is the interval, in seconds, between node polling.
 	NodeControllerPollingInterval int
+
+	// Kubelet directory provides the operation of volumeAttributesClassName.
+	KubeletDir string
 }
 
 // Default returns a new instance of config

pkg/driver/controller.go

@@ -366,14 +366,22 @@ func CreateLVMVolume(ctx context.Context, req *csi.CreateVolumeRequest,
 	}
 	klog.Infof("scheduling the volume %s/%s on node %s",
 		params.VgPattern.String(), volName, owner)
+
+	qos, err := NewQoSParams(req.GetMutableParameters())
+	if err != nil {
+		return nil, err
+	}
+
 	volObj, err := volbuilder.NewBuilder().
 		WithName(volName).
 		WithCapacity(capacity).
 		WithVgPattern(params.VgPattern.String()).
 		WithOwnerNode(owner).
 		WithVolumeStatus(lvm.LVMStatusPending).
 		WithShared(params.Shared).
-		WithThinProvision(params.ThinProvision).Build()
+		WithThinProvision(params.ThinProvision).
+		WithQoS(QoSParamsCreateVolume(qos)).
+		Build()
 
 	if err != nil {
 		return nil, status.Errorf(codes.Internal,
@@ -1103,6 +1111,7 @@ func newControllerCapabilities() []*csi.ControllerServiceCapability {
 		csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
 		csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
 		csi.ControllerServiceCapability_RPC_GET_CAPACITY,
+		csi.ControllerServiceCapability_RPC_MODIFY_VOLUME,
 	} {
 		capabilities = append(capabilities, fromType(cap))
 	}
@@ -1255,9 +1264,50 @@ func (cs *controller) ControllerGetVolume(
 	return nil, status.Error(codes.Unimplemented, "ControllerGetVolume is not implemented")
 }
 
+// ControllerModifyVolume update previously provisioned VAC parametrs
+//
+// This implements csi.ControllerServer
 func (cs *controller) ControllerModifyVolume(
 	ctx context.Context,
 	req *csi.ControllerModifyVolumeRequest,
 ) (*csi.ControllerModifyVolumeResponse, error) {
-	return nil, status.Error(codes.Unimplemented, "ControllerModifyVolume is not implemented")
+	if req == nil {
+		return nil, status.Error(codes.InvalidArgument, "request is nil")
+	}
+	volID := strings.ToLower(strings.TrimSpace(req.GetVolumeId()))
+	if volID == "" {
+		return nil, status.Error(codes.InvalidArgument, "volume_id is empty")
+	}
+
+	vol, err := volbuilder.NewKubeclient().
+		WithNamespace(lvm.LvmNamespace).
+		Get(volID, metav1.GetOptions{})
+	if err != nil {
+		if k8serror.IsNotFound(err) {
+			return nil, status.Errorf(codes.NotFound, "volume %s not found", volID)
+		}
+		return nil, status.Errorf(codes.Internal, "failed to get lvmvolume %s: %v", volID, err)
+	}
+
+	// get the VAC values and prepare lvmvolume spec io format
+	ioPatch, err := NewQoSParams(req.GetMutableParameters())
+	if err != nil {
+		return nil, err
+	}
+
+	// apply new io values
+	modify, changed := QoSParamsModifyVolume(ioPatch, vol.Spec.QoS)
+	if !changed {
+		klog.Infof("ModifyVolume: QoS unchanged for volume %v, skipping update", vol)
+		return &csi.ControllerModifyVolumeResponse{}, nil
+	}
+	vol.Spec.QoS = modify
+
+	if _, err := volbuilder.NewKubeclient().
+		WithNamespace(lvm.LvmNamespace).
+		Update(vol); err != nil {
+		return nil, status.Errorf(codes.Internal, "failed to update lvmvolume qos %s: %v", volID, err)
+	}
+
+	return &csi.ControllerModifyVolumeResponse{}, nil
 }