cpu: add e2e test for cstates

Author: askervin

test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/balloons-cstates.cfg

@@ -0,0 +1,30 @@
+config:
+  agent:
+    nodeResourceTopology: true
+  allocatorTopologyBalancing: false
+  availableResources:
+    cpuset: 2-7,10-13 # policy must not touch C-states of unavailable CPUs
+  reservedResources:
+    cpu: 750m
+
+  pinCPU: true
+
+  idleCPUClass: default-class
+
+  balloonTypes:
+  - name: lowlatency-bln
+    cpuClass: lowlatency-class
+
+  control:
+    cpu:
+      classes:
+        lowlatency-class:
+          disabledCstates: [C4, C6, C8, C10]
+        default-class:
+          disabledCstates: []
+  log:
+    debug:
+      - policy
+      - cpu
+extraEnv:
+  OVERRIDE_SYS_CSTATES: '''[{"cpus": "0-15", "names": ["C1E", "C2", "C4", "C8"], "files": {"disable": "0"}}]'''

test/e2e/policies.test-suite/balloons/n4c16/test17-cstates/code.var.sh

@@ -0,0 +1,81 @@
+# Test balloons with certain CPU C-states disabled
+
+helm-terminate
+helm_config=$TEST_DIR/balloons-cstates.cfg helm-launch balloons
+
+# cpuids-of-container returns CPU ids a container is allowed to use, e.g. "1 2 4"
+cpuids-of() {
+    local ctr=$1 # e.g. pod0c0
+    pyexec "for cpu in cpus['$ctr']: print(cpu.replace('cpu0',''))"
+}
+
+# verify-cstates checks the last writes to "disable" files in the
+# override fs.
+verify-cstates() {
+    local cpu_ids=$1          # e.g. "1 2 4"
+    local enabled_cstates=$2  # e.g. "C1E C2"
+    local disabled_cstates=$3 # e.g. "C6 C8"
+    local last_n_writes=$4    # expect the write within last N writes, e.g. 6
+
+    vm-command "kubectl -n kube-system logs ds/nri-resource-policy-balloons | nl | grep 'cstates override fs: wrote' | tail -n $last_n_writes"
+    for cpu_id in $cpu_ids; do
+        for cstate in $enabled_cstates; do
+            echo "verify last write to cpu$cpu_id cstate=$cstate disable is 0"
+            grep "cpu${cpu_id} cstate=$cstate disable=" <<< $COMMAND_OUTPUT | tail -n 1 | grep -q 'disable="0"' || {
+                command-error "expected write 0 not found"
+            }
+        done
+        for cstate in $disabled_cstates; do
+            echo "verify last write to cpu$cpu_id cstate=$cstate disable is 1"
+            grep "cpu${cpu_id} cstate=$cstate disable=" <<< $COMMAND_OUTPUT | tail -n 1 | grep -q 'disable="1"' || {
+                command-error "expected write 1 not found"
+            }
+        done
+    done
+}
+
+cleanup() {
+    vm-command "kubectl delete pods --all --now"
+}
+
+CPUREQ="750m" MEMREQ="100M" CPULIM="750m" MEMLIM=""
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: lowlatency-bln" CONTCOUNT=1 create balloons-busybox
+report allowed
+verify 'len(cpus["pod0c0"]) == 1'
+verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 4
+
+
+CPUREQ="3" MEMREQ="100M" CPULIM="" MEMLIM=""
+POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: lowlatency-bln" CONTCOUNT=1 create balloons-busybox
+report allowed
+verify 'cpus["pod0c0"] == cpus["pod1c0"]' \
+       'len(cpus["pod0c0"]) == 4'
+verify-cstates "$(cpuids-of pod1c0)" "C1E C2" "C4 C8" 16
+
+max_lowlatency_cpus=$(cpuids-of pod1c0)
+
+vm-command 'kubectl delete pod pod1'
+report allowed
+verify 'len(cpus["pod0c0"]) == 1'
+
+pod0cpus=" $(echo $(cpuids-of pod0c0) ) "
+
+echo "verify that C-states of freed CPUs are enabled again after balloon was deflated"
+freed_cpus=""
+for cpu_id in $max_lowlatency_cpus; do
+    grep -q " $cpu_id " <<< $pod0cpus || freed_cpus+=" $cpu_id"
+done
+echo "verify that all C-states of freed CPUs $freed_cpus (= {$max_lowlatency_cpus} - {$pod0cpus}) are enabled again after balloon was deflated"
+verify-cstates "$freed_cpus" "C1E C2 C4 C8" "" 32
+
+echo "verify that C-states of remaining CPUs $(cpuids-of pod0c0) are still configured for low-latency"
+verify-cstates "$(cpuids-of pod0c0)" "C1E C2" "C4 C8" 18
+
+vm-command 'kubectl delete pod pod0'
+report allowed
+
+# verify that after all containers are gone, all C-states of all their CPUs are restored
+verify-cstates "$max_lowlatency_cpus" "C1E C2 C4 C8" "" 32
+
+breakpoint
+exit 1 # FORCED FAIL