[WIP] implement autoscaling

api/v1/schedulingconfigtemplate_types.go

@@ -86,17 +86,75 @@ type GPUFilter struct {
 }
 
 type AutoScalingConfig struct {
-	// layer 1 vertical auto-scaling, turbo burst to existing GPU cards quickly
-	// VPA-like, aggregate metrics data <1m
-	AutoSetLimits AutoSetLimits `json:"autoSetLimits,omitempty"`
+	// layer 1 adjusting, to match the actual usage in the long run, only for N:M remote vGPU mode
+	// Adjust baseline requests to match the actual usage in longer period, such as 1day - 2weeks
+	AutoSetResources AutoSetResources `json:"autoSetResources,omitempty"`
 
 	// layer 2 horizontal auto-scaling, scale up to more GPU cards if max limits threshold hit
 	// HPA-like, aggregate metrics data 1m-1h (when tf-worker scaled-up, should also trigger client pod's owner[Deployment etc.]'s replica increasing, check if KNative works)
 	AutoSetReplicas AutoSetReplicas `json:"autoSetReplicas,omitempty"`
 
-	// layer 3 adjusting, to match the actual usage in the long run, only for N:M remote vGPU mode, not impl yet
-	// Adjust baseline requests to match the actual usage in longer period, such as 1day - 2weeks
-	AutoSetRequests AutoSetRequests `json:"autoSetRequests,omitempty"`
+	// CronScalingRules defines a list of CronScaling rules used to schedule scaling actions based on cron expressions.
+	CronScalingRules []CronScalingRule `json:"cronScalingRules,omitempty"`
+}
+
+// CronScalingRule defines the rule for scaling resources based on a cron schedule.
+// It allows enabling/disabling the scaler, specifying the time window for scaling,
+// and configuring the desired resources and replicas during the scheduled period.
+type CronScalingRule struct {
+	// Enable specifies whether the cron scaler is enabled.
+	Enable bool `json:"enable,omitempty"`
+	// Name is the identifier for the cron scaler.
+	Name string `json:"name,omitempty"`
+	// Start is the start time for the scaling schedule, in cron format.
+	Start string `json:"start,omitempty"`
+	// End is the end time for the scaling schedule, in cron format.
+	End string `json:"end,omitempty"`
+	// DesiredResources specifies the target resources to scale to during the schedule.
+	DesiredResources Resources `json:"desiredResources,omitempty"`
+	// ResourceMultiplier is a string representing the multiplier to apply to resources.
+	ResourceMultiplier string `json:"resourceMultiplier,omitempty"`
+	// DesiredReplicas is the target number of replicas during the schedule.
+	DesiredReplicas *int32 `json:"desiredReplicas,omitempty"`
+	// ReplicasMultiplier is a string representing the multiplier to apply to replicas.
+	ReplicasMultiplier string `json:"replicasMultiplier,omitempty"`
+}
+
+type AutoSetResources struct {
+	Enable bool `json:"enable,omitempty"`
+
+	// Target resource to scale, such as "tflops", "vram", or "all" by default
+	TargetResource string `json:"targetResource,omitempty"`
+
+	// Tflops usage percentile that will be used as a base for tflops target recommendation. Default: 0.9
+	TargetTflopsPercentile string `json:"targettflopspercentile,omitempty"`
+
+	// Tflops usage percentile that will be used for the lower bound on tflops recommendation. Default: 0.5
+	LowerBoundTflopsPercentile string `json:"lowerboundtflopspercentile,omitempty"`
+
+	// Tflops usage percentile that will be used for the upper bound on tflops recommendation. Default: 0.95
+	UpperBoundTflopsPercentile string `json:"upperboundtflopspercentile,omitempty"`
+
+	// Vram usage percentile that will be used as a base for vram target recommendation. Default: 0.9
+	TargetVramPercentile string `json:"targetvrampercentile,omitempty"`
+
+	// Vram usage percentile that will be used for the lower bound on vram recommendation. Default: 0.5
+	LowerBoundVramPercentile string `json:"lowerboundvrampercentile,omitempty"`
+
+	// Vram usage percentile that will be used for the upper bound on vram recommendation. Default: 0.95
+	UpperBoundVramPercentile string `json:"upperboundvrampercentile,omitempty"`
+
+	// Fraction of usage added as the safety margin to the recommended request. Default: 0.15
+	RequestMarginFraction string `json:"requestMarginFraction,omitempty"`
+
+	// The time interval used for computing the confidence multiplier for the lower and upper bound. Default: 24h
+	ConfidenceInterval string `json:"confidenceInterval,omitempty"`
+
+	// How much time back TSDB have to be queried to get historical metrics. Default: 1d
+	HistoryLength string `json:"historyLength,omitempty"`
+
+	// Resolution at which TSDB is queried for historical metrics. Default: 1m
+	HistoryResolution string `json:"historyResolution,omitempty"`
 }
 
 // A typical autoLimits algorithm could be checking every 5m, look back 1 day data,

api/v1/tensorfusionconnection_types.go

@@ -21,6 +21,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+type ResourceName string
+
+const (
+	ResourceTflops ResourceName = "tflops"
+	ResourceVram   ResourceName = "vram"
+)
+
 type Resource struct {
 	Tflops resource.Quantity `json:"tflops"`
 	Vram   resource.Quantity `json:"vram"`
@@ -31,6 +38,20 @@ type Resources struct {
 	Limits   Resource `json:"limits"`
 }
 
+func (r *Resources) Equal(t *Resources) bool {
+	return r.Requests.Tflops.Equal(t.Requests.Tflops) &&
+		r.Requests.Vram.Equal(t.Requests.Vram) &&
+		r.Limits.Tflops.Equal(t.Limits.Tflops) &&
+		r.Limits.Vram.Equal(t.Limits.Vram)
+}
+
+func (r *Resources) IsZero() bool {
+	return r.Requests.Tflops.IsZero() &&
+		r.Requests.Vram.IsZero() &&
+		r.Limits.Tflops.IsZero() &&
+		r.Limits.Vram.IsZero()
+}
+
 // TensorFusionConnectionSpec defines the desired state of TensorFusionConnection.
 type TensorFusionConnectionSpec struct {
 	WorkloadName string `json:"workloadName"`

charts/tensor-fusion/crds/tensor-fusion.ai_schedulingconfigtemplates.yaml

@@ -50,41 +50,6 @@ spec:
               autoScaling:
                 description: scale the workload based on the usage and traffic
                 properties:
-                  autoSetLimits:
-                    description: |-
-                      layer 1 vertical auto-scaling, turbo burst to existing GPU cards quickly
-                      VPA-like, aggregate metrics data <1m
-                    properties:
-                      enable:
-                        type: boolean
-                      evaluationPeriod:
-                        type: string
-                      extraTFlopsBufferRatio:
-                        type: string
-                      ignoredDeltaRange:
-                        type: string
-                      maxRatioToRequests:
-                        description: the multiplier of requests, to avoid limit set
-                          too high, like 5.0
-                        type: string
-                      prediction:
-                        properties:
-                          enable:
-                            type: boolean
-                          historyDataPeriod:
-                            type: string
-                          model:
-                            type: string
-                          predictionPeriod:
-                            type: string
-                        type: object
-                      scaleUpStep:
-                        type: string
-                      targetResource:
-                        description: target resource to scale limits, such as "tflops",
-                          "vram", or "all" by default
-                        type: string
-                    type: object
                   autoSetReplicas:
                     description: |-
                       layer 2 horizontal auto-scaling, scale up to more GPU cards if max limits threshold hit
@@ -105,40 +70,141 @@ spec:
                       targetTFlopsOfLimits:
                         type: string
                     type: object
-                  autoSetRequests:
+                  autoSetResources:
                     description: |-
-                      layer 3 adjusting, to match the actual usage in the long run, only for N:M remote vGPU mode, not impl yet
+                      layer 1 adjusting, to match the actual usage in the long run, only for N:M remote vGPU mode
                       Adjust baseline requests to match the actual usage in longer period, such as 1day - 2weeks
                     properties:
-                      aggregationPeriod:
+                      confidenceInterval:
+                        description: 'The time interval used for computing the confidence
+                          multiplier for the lower and upper bound. Default: 24h'
                         type: string
                       enable:
                         type: boolean
-                      evaluationPeriod:
+                      historyLength:
+                        description: 'How much time back TSDB have to be queried to
+                          get historical metrics. Default: 1d'
                         type: string
-                      extraBufferRatio:
-                        description: the request buffer ratio, for example actual
-                          usage is 1.0, 10% buffer will be 1.1 as final preferred
-                          requests
+                      historyResolution:
+                        description: 'Resolution at which TSDB is queried for historical
+                          metrics. Default: 1m'
                         type: string
-                      percentileForAutoRequests:
+                      lowerboundtflopspercentile:
+                        description: 'Tflops usage percentile that will be used for
+                          the lower bound on tflops recommendation. Default: 0.5'
+                        type: string
+                      lowerboundvrampercentile:
+                        description: 'Vram usage percentile that will be used for
+                          the lower bound on vram recommendation. Default: 0.5'
+                        type: string
+                      requestMarginFraction:
+                        description: 'Fraction of usage added as the safety margin
+                          to the recommended request. Default: 0.15'
                         type: string
-                      prediction:
-                        properties:
-                          enable:
-                            type: boolean
-                          historyDataPeriod:
-                            type: string
-                          model:
-                            type: string
-                          predictionPeriod:
-                            type: string
-                        type: object
                       targetResource:
-                        description: target resource to scale requests, such as "tflops",
-                          "vram", or "all" by default
+                        description: Target resource to scale, such as "tflops", "vram",
+                          or "all" by default
+                        type: string
+                      targettflopspercentile:
+                        description: 'Tflops usage percentile that will be used as
+                          a base for tflops target recommendation. Default: 0.9'
+                        type: string
+                      targetvrampercentile:
+                        description: 'Vram usage percentile that will be used as a
+                          base for vram target recommendation. Default: 0.9'
+                        type: string
+                      upperboundtflopspercentile:
+                        description: 'Tflops usage percentile that will be used for
+                          the upper bound on tflops recommendation. Default: 0.95'
+                        type: string
+                      upperboundvrampercentile:
+                        description: 'Vram usage percentile that will be used for
+                          the upper bound on vram recommendation. Default: 0.95'
                         type: string
                     type: object
+                  cronScalingRules:
+                    description: CronScalingRules defines a list of CronScaling rules
+                      used to schedule scaling actions based on cron expressions.
+                    items:
+                      description: |-
+                        CronScalingRule defines the rule for scaling resources based on a cron schedule.
+                        It allows enabling/disabling the scaler, specifying the time window for scaling,
+                        and configuring the desired resources and replicas during the scheduled period.
+                      properties:
+                        desiredReplicas:
+                          description: DesiredReplicas is the target number of replicas
+                            during the schedule.
+                          format: int32
+                          type: integer
+                        desiredResources:
+                          description: DesiredResources specifies the target resources
+                            to scale to during the schedule.
+                          properties:
+                            limits:
+                              properties:
+                                tflops:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                vram:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                              required:
+                              - tflops
+                              - vram
+                              type: object
+                            requests:
+                              properties:
+                                tflops:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                vram:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                              required:
+                              - tflops
+                              - vram
+                              type: object
+                          required:
+                          - limits
+                          - requests
+                          type: object
+                        enable:
+                          description: Enable specifies whether the cron scaler is
+                            enabled.
+                          type: boolean
+                        end:
+                          description: End is the end time for the scaling schedule,
+                            in cron format.
+                          type: string
+                        name:
+                          description: Name is the identifier for the cron scaler.
+                          type: string
+                        replicasMultiplier:
+                          description: ReplicasMultiplier is a string representing
+                            the multiplier to apply to replicas.
+                          type: string
+                        resourceMultiplier:
+                          description: ResourceMultiplier is a string representing
+                            the multiplier to apply to resources.
+                          type: string
+                        start:
+                          description: Start is the start time for the scaling schedule,
+                            in cron format.
+                          type: string
+                      type: object
+                    type: array
                 type: object
               hypervisor:
                 description: single GPU device multi-process queuing and fair scheduling