Skip to content

Commit e3a83fd

Browse files
kamarabbas99kamarabbas99
committed
Add e2e tests for CPU startup boost
1 parent 32c68bd commit e3a83fd

File tree

4 files changed

+375
-5
lines changed

4 files changed

+375
-5
lines changed

‎vertical-pod-autoscaler/e2e/v1/admission_controller.go‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
apiv1 "k8s.io/api/core/v1"
2929
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3030
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
31+
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
3132
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
3233
"k8s.io/kubernetes/test/e2e/framework"
3334
framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment"
@@ -961,6 +962,123 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
961962
})
962963
})
963964

965+
var _ = AdmissionControllerE2eDescribe("Admission-controller", ginkgo.Label("FG:CPUStartupBoost"), func() {
966+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
967+
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
968+
969+
ginkgo.BeforeEach(func() {
970+
checkFeatureGateTestsEnabled(f, features.CPUStartupBoost, true, false)
971+
waitForVpaWebhookRegistration(f)
972+
})
973+
974+
ginkgo.It("boosts CPU by factor on pod creation", func() {
975+
initialCPU := ParseQuantityOrDie("100m")
976+
expectedCPU := ParseQuantityOrDie("200m")
977+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
978+
979+
ginkgo.By("Setting up a VPA with a startup boost policy (factor)")
980+
containerName := GetHamsterContainerNameByIndex(0)
981+
factor := int32(2)
982+
vpaCRD := test.VerticalPodAutoscaler().
983+
WithName("hamster-vpa").
984+
WithNamespace(f.Namespace.Name).
985+
WithTargetRef(hamsterTargetRef).
986+
WithContainer(containerName).
987+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
988+
AppendRecommendation(
989+
test.Recommendation().
990+
WithContainer(containerName).
991+
WithTarget("100m", "100Mi").
992+
GetContainerResources(),
993+
).
994+
Get()
995+
InstallVPA(f, vpaCRD)
996+
997+
ginkgo.By("Starting the deployment and verifying the pod is boosted")
998+
podList := startDeploymentPods(f, d)
999+
pod := podList.Items[0]
1000+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1001+
gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1002+
})
1003+
1004+
ginkgo.It("boosts CPU by quantity on pod creation", func() {
1005+
initialCPU := ParseQuantityOrDie("100m")
1006+
boostCPUQuantity := ParseQuantityOrDie("500m")
1007+
expectedCPU := ParseQuantityOrDie("600m")
1008+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1009+
1010+
ginkgo.By("Setting up a VPA with a startup boost policy (quantity)")
1011+
containerName := GetHamsterContainerNameByIndex(0)
1012+
vpaCRD := test.VerticalPodAutoscaler().
1013+
WithName("hamster-vpa").
1014+
WithNamespace(f.Namespace.Name).
1015+
WithTargetRef(hamsterTargetRef).
1016+
WithContainer(containerName).
1017+
WithCPUStartupBoost(vpa_types.QuantityStartupBoostType, nil, &boostCPUQuantity, "15s").
1018+
AppendRecommendation(
1019+
test.Recommendation().
1020+
WithContainer(containerName).
1021+
WithTarget("100m", "100Mi").
1022+
GetContainerResources(),
1023+
).
1024+
Get()
1025+
InstallVPA(f, vpaCRD)
1026+
1027+
ginkgo.By("Starting the deployment and verifying the pod is boosted")
1028+
podList := startDeploymentPods(f, d)
1029+
pod := podList.Items[0]
1030+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1031+
gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1032+
})
1033+
1034+
ginkgo.It("boosts CPU on pod creation when VPA update mode is Off", func() {
1035+
initialCPU := ParseQuantityOrDie("100m")
1036+
expectedCPU := ParseQuantityOrDie("200m")
1037+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1038+
1039+
ginkgo.By("Setting up a VPA with updateMode Off and a startup boost policy")
1040+
containerName := GetHamsterContainerNameByIndex(0)
1041+
factor := int32(2)
1042+
vpaCRD := test.VerticalPodAutoscaler().
1043+
WithName("hamster-vpa").
1044+
WithNamespace(f.Namespace.Name).
1045+
WithTargetRef(hamsterTargetRef).
1046+
WithContainer(containerName).
1047+
WithUpdateMode(vpa_types.UpdateModeOff). // VPA is off, but boost should still work
1048+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
1049+
Get()
1050+
InstallVPA(f, vpaCRD)
1051+
1052+
ginkgo.By("Starting the deployment and verifying the pod is boosted")
1053+
podList := startDeploymentPods(f, d)
1054+
pod := podList.Items[0]
1055+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1056+
})
1057+
1058+
ginkgo.It("doesn't boost CPU on pod creation when scaling mode is Off", func() {
1059+
initialCPU := ParseQuantityOrDie("100m")
1060+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1061+
1062+
ginkgo.By("Setting up a VPA with a startup boost policy and scaling mode Off")
1063+
containerName := GetHamsterContainerNameByIndex(0)
1064+
factor := int32(2)
1065+
vpaCRD := test.VerticalPodAutoscaler().
1066+
WithName("hamster-vpa").
1067+
WithNamespace(f.Namespace.Name).
1068+
WithTargetRef(hamsterTargetRef).
1069+
WithContainer(containerName).
1070+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
1071+
WithScalingMode(containerName, vpa_types.ContainerScalingModeOff).
1072+
Get()
1073+
InstallVPA(f, vpaCRD)
1074+
1075+
ginkgo.By("Starting the deployment and verifying the pod is NOT boosted")
1076+
podList := startDeploymentPods(f, d)
1077+
pod := podList.Items[0]
1078+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(initialCPU)).To(gomega.Equal(0))
1079+
})
1080+
})
1081+
9641082
func startDeploymentPods(f *framework.Framework, deployment *appsv1.Deployment) *apiv1.PodList {
9651083
// Apiserver watch can lag depending on cached object count and apiserver resource usage.
9661084
// We assume that watch can lag up to 5 seconds.

‎vertical-pod-autoscaler/e2e/v1/common.go‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
4040
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
4141
clientset "k8s.io/client-go/kubernetes"
42+
"k8s.io/component-base/featuregate"
4243
"k8s.io/kubernetes/test/e2e/framework"
4344
framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment"
4445
)
@@ -359,14 +360,30 @@ func PatchVpaRecommendation(f *framework.Framework, vpa *vpa_types.VerticalPodAu
359360

360361
// AnnotatePod adds annotation for an existing pod.
361362
func AnnotatePod(f *framework.Framework, podName, annotationName, annotationValue string) {
362-
bytes, err := json.Marshal([]patchRecord{{
363+
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), podName, metav1.GetOptions{})
364+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get pod.")
365+
366+
patches := []patchRecord{}
367+
if pod.Annotations == nil {
368+
patches = append(patches, patchRecord{
369+
Op: "add",
370+
Path: "/metadata/annotations",
371+
Value: make(map[string]string),
372+
})
373+
}
374+
375+
patches = append(patches, patchRecord{
363376
Op: "add",
364-
Path: fmt.Sprintf("/metadata/annotations/%v", annotationName),
377+
Path: fmt.Sprintf("/metadata/annotations/%s", annotationName),
365378
Value: annotationValue,
366-
}})
367-
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{})
379+
})
380+
381+
bytes, err := json.Marshal(patches)
382+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
383+
384+
patchedPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{})
368385
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch pod.")
369-
gomega.Expect(pod.Annotations[annotationName]).To(gomega.Equal(annotationValue))
386+
gomega.Expect(patchedPod.Annotations[annotationName]).To(gomega.Equal(annotationValue))
370387
}
371388

372389
// ParseQuantityOrDie parses quantity from string and dies with an error if
@@ -654,3 +671,36 @@ func anyContainsSubstring(arr []string, substr string) bool {
654671
}
655672
return false
656673
}
674+
675+
// checkFeatureGateTestsEnabled check for enabled feature gates in the cluster used for the
676+
// given VPA feature.
677+
// Use this in a "beforeEach" call before any suites that use a featuregate.
678+
func checkFeatureGateTestsEnabled(f *framework.Framework, feature featuregate.Feature, checkAdmission, checkUpdater bool) {
679+
ginkgo.By(fmt.Sprintf("Checking %s cluster feature gate is on", feature))
680+
681+
if checkUpdater {
682+
ginkgo.By(fmt.Sprintf("Checking %s VPA feature gate is enabled for updater", feature))
683+
684+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
685+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
686+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
687+
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
688+
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
689+
if !anyContainsSubstring(vpaUpdaterPod.Args, fmt.Sprintf("%s=true", string(feature))) {
690+
ginkgo.Skip(fmt.Sprintf("Skipping suite: %s feature gate is not enabled for the VPA updater", feature))
691+
}
692+
}
693+
694+
if checkAdmission {
695+
ginkgo.By(fmt.Sprintf("Checking %s VPA feature gate is enabled for admission controller", feature))
696+
697+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-admission-controller", metav1.GetOptions{})
698+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
699+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
700+
vpaAdmissionPod := deploy.Spec.Template.Spec.Containers[0]
701+
gomega.Expect(vpaAdmissionPod.Name).To(gomega.Equal("admission-controller"))
702+
if !anyContainsSubstring(vpaAdmissionPod.Args, fmt.Sprintf("%s=true", string(feature))) {
703+
ginkgo.Skip(fmt.Sprintf("Skipping suite: %s feature gate is not enabled for VPA admission controller", feature))
704+
}
705+
}
706+
}

‎vertical-pod-autoscaler/e2e/v1/full_vpa.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2828
"k8s.io/apimachinery/pkg/util/wait"
2929
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
30+
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
3031
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/utils/test"
3132
"k8s.io/kubernetes/test/e2e/framework"
3233
podsecurity "k8s.io/pod-security-admission/api"
@@ -347,6 +348,122 @@ var _ = FullVpaE2eDescribe("Pods under VPA with non-recognized recommender expli
347348
})
348349
})
349350

351+
var _ = FullVpaE2eDescribe("Pods under VPA with CPUStartupBoost", ginkgo.Label("FG:CPUStartupBoost"), func() {
352+
var (
353+
rc *ResourceConsumer
354+
)
355+
replicas := 3
356+
357+
ginkgo.AfterEach(func() {
358+
rc.CleanUp()
359+
})
360+
361+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
362+
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
363+
364+
ginkgo.Describe("have CPU startup boost recommendation applied", func() {
365+
ginkgo.BeforeEach(func() {
366+
checkFeatureGateTestsEnabled(f, features.CPUStartupBoost, true, true)
367+
waitForVpaWebhookRegistration(f)
368+
})
369+
370+
ginkgo.It("to all containers of a pod", func() {
371+
ns := f.Namespace.Name
372+
ginkgo.By("Setting up a VPA CRD with CPUStartupBoost")
373+
targetRef := &autoscaling.CrossVersionObjectReference{
374+
APIVersion: "apps/v1",
375+
Kind: "Deployment",
376+
Name: "hamster",
377+
}
378+
379+
containerName := GetHamsterContainerNameByIndex(0)
380+
factor := int32(100)
381+
vpaCRD := test.VerticalPodAutoscaler().
382+
WithName("hamster-vpa").
383+
WithNamespace(f.Namespace.Name).
384+
WithTargetRef(targetRef).
385+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
386+
WithContainer(containerName).
387+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s").
388+
Get()
389+
InstallVPA(f, vpaCRD)
390+
391+
ginkgo.By("Setting up a hamster deployment")
392+
rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment,
393+
replicas,
394+
1, /*initCPUTotal*/
395+
10, /*initMemoryTotal*/
396+
1, /*initCustomMetric*/
397+
initialCPU, /*cpuRequest*/
398+
initialMemory, /*memRequest*/
399+
f.ClientSet,
400+
f.ScalesGetter)
401+
402+
// Pods should be created with boosted CPU (10m * 100 = 1000m)
403+
err := waitForResourceRequestInRangeInPods(
404+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
405+
ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m"))
406+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
407+
408+
// Pods should be scaled back down in-place after they become Ready and
409+
// StartupBoost.CPU.Duration has elapsed
410+
err = waitForResourceRequestInRangeInPods(
411+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
412+
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
413+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
414+
})
415+
416+
ginkgo.It("to a subset of containers in a pod", func() {
417+
ns := f.Namespace.Name
418+
419+
ginkgo.By("Setting up a VPA CRD with CPUStartupBoost")
420+
targetRef := &autoscaling.CrossVersionObjectReference{
421+
APIVersion: "apps/v1",
422+
Kind: "Deployment",
423+
Name: "hamster",
424+
}
425+
426+
containerName := GetHamsterContainerNameByIndex(0)
427+
factor := int32(100)
428+
vpaCRD := test.VerticalPodAutoscaler().
429+
WithName("hamster-vpa").
430+
WithNamespace(f.Namespace.Name).
431+
WithTargetRef(targetRef).
432+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
433+
WithContainer(containerName).
434+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s").
435+
Get()
436+
437+
InstallVPA(f, vpaCRD)
438+
439+
ginkgo.By("Setting up a hamster deployment")
440+
rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment,
441+
replicas,
442+
1, /*initCPUTotal*/
443+
10, /*initMemoryTotal*/
444+
1, /*initCustomMetric*/
445+
initialCPU, /*cpuRequest*/
446+
initialMemory, /*memRequest*/
447+
f.ClientSet,
448+
f.ScalesGetter)
449+
450+
// Pods should be created with boosted CPU (10m * 100 = 1000m)
451+
err := waitForResourceRequestInRangeInPods(
452+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
453+
ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m"))
454+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
455+
456+
// Pods should be scaled back down in-place after they become Ready and
457+
// StartupBoost.CPU.Duration has elapsed
458+
err = waitForResourceRequestInRangeInPods(
459+
f, pollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
460+
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
461+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
462+
})
463+
})
464+
465+
})
466+
350467
var _ = FullVpaE2eDescribe("OOMing pods under VPA", func() {
351468
const replicas = 3
352469

0 commit comments

Comments
 (0)