From 3c2aaba86c984e04991d16909d2111b112c61442 Mon Sep 17 00:00:00 2001 From: cah-patrickthiem Date: Wed, 28 Aug 2024 17:27:39 +0200 Subject: [PATCH 1/6] Added the first robustness feature test called maxRequestInflight. There is a positive and a negative test case. --- .../scs_0215_v1_robustness_features_test.go | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go new file mode 100644 index 000000000..7d0b011e7 --- /dev/null +++ b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go @@ -0,0 +1,118 @@ +package scs_k8s_tests + +import ( + "context" + "fmt" + "log" + "os" + "strconv" + "sync" + "testing" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// getClusterSize estimates the cluster size by counting the number of nodes. +func getClusterSize(clientset *kubernetes.Clientset) int { + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{}) + if err != nil { + log.Fatalf("Failed to list nodes: %v", err) + } + return len(nodes.Items) +} + +// runConcurrentRequests sends concurrent API requests and returns the number of errors encountered. +func runConcurrentRequests(clientset *kubernetes.Clientset, maxRequestsInflight int) int { + var wg sync.WaitGroup + errChan := make(chan error, maxRequestsInflight) + + for i := 0; i < maxRequestsInflight; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) + defer cancel() + _, err := clientset.CoreV1().Pods("").List(ctx, v1.ListOptions{}) + if err != nil { + errChan <- err + } + }() + } + + wg.Wait() + close(errChan) + return len(errChan) // Return the number of errors encountered +} + +// testPositiveCase handles the positive scenario where requests should succeed. +func testPositiveCase(t *testing.T, clientset *kubernetes.Clientset, maxRequestsInflight int) { + fmt.Println("Running Positive Test Case") + // Reduce the load even further for the positive test (e.g., 25% of maxRequestsInflight) + safeRequests := maxRequestsInflight / 4 + errors := runConcurrentRequests(clientset, safeRequests) + if errors > 0 { + t.Errorf("Test failed: encountered %d unexpected errors when requests were expected to succeed.", errors) + } else { + fmt.Println("Positive test case passed successfully!") + } +} + +// testNegativeCase handles the negative scenario where requests should be throttled. +func testNegativeCase(t *testing.T, clientset *kubernetes.Clientset, maxRequestsInflight int) { + fmt.Println("Running Negative Test Case") + // Increase the load significantly above the maxRequestsInflight to trigger rate limiting + overloadRequests := maxRequestsInflight * 2 + errors := runConcurrentRequests(clientset, overloadRequests) + + // Expect at least some errors due to rate limiting + if errors == 0 { + t.Errorf("Test failed: expected rate limit errors, but all requests succeeded.") + } else { + fmt.Println("Negative test case passed as expected: rate limit exceeded.") + } +} + +// Test_scs_maxRequestInflight is the main entry point that runs both positive and negative test cases. +func Test_scs_0215_maxRequestInflight(t *testing.T) { + // Load in-cluster configuration + config, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("Failed to load in-cluster config: %v", err) + } + + // Adjust client rate limits + config.QPS = 50 // Queries Per Second + config.Burst = 100 // Allowed burst (concurrent requests above QPS) + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create Kubernetes client: %v", err) + } + + + // Get cluster size (number of nodes) + clusterSize := getClusterSize(clientset) + fmt.Printf("Detected cluster size: %d nodes\n", clusterSize) + + // Determine maxRequestsInflight based on cluster size and environment variable + maxRequestsInflightStr := os.Getenv("MAX_REQUESTS_INFLIGHT") + maxRequestsInflight, err := strconv.Atoi(maxRequestsInflightStr) + if err != nil || maxRequestsInflight <= 0 { + maxRequestsInflight = clusterSize * 250 // Example scaling logic: 100 requests per node + } + + fmt.Printf("Using maxRequestsInflight = %d\n", maxRequestsInflight) + + // Run the positive test case + t.Run("Positive Test Case", func(t *testing.T) { + testPositiveCase(t, clientset, maxRequestsInflight) + }) + + // Run the negative test case + t.Run("Negative Test Case", func(t *testing.T) { + testNegativeCase(t, clientset, maxRequestsInflight) + }) +} From 4d1160221eab2d209b13f2f884ba9ef82054e9e1 Mon Sep 17 00:00:00 2001 From: cah-patrickthiem Date: Thu, 12 Sep 2024 14:57:36 +0200 Subject: [PATCH 2/6] Added two more tests regarding the robustness K8s standard, namely "maxMutatingRequestInflight" and "minRequestTimeout". --- .../Makefile | 132 ++++++++ .../kind-config.yaml | 19 ++ .../event-ratelimit-config.yaml | 8 + .../scs_0215_v1_robustness_features_test.go | 298 +++++++++++++++--- 4 files changed, 410 insertions(+), 47 deletions(-) create mode 100644 Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile create mode 100644 Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml create mode 100644 Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml diff --git a/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile b/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile new file mode 100644 index 000000000..2892164bb --- /dev/null +++ b/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile @@ -0,0 +1,132 @@ +# Makefile +# This makefile is for development purpose + +SHELL = /bin/bash +#SED ?= sed + +DOCKERFILE="Dockerfile" +IMAGE_REGISTRY="ghcr.io/sovereigncloudstack/standards" +IMAGE_NAME="scsconformance" +#IMAGE_VERSION_TAG ="v0.1.2" + +KIND_CLUSTER ="testcluster" + +#PLUGIN_NAME="k8s-default-storage-class-plugin-go" +PLUGIN_NAME="plugin" +PLUGIN_FILE="${PLUGIN_NAME}.yaml" + +#~ SONO_WAIT = 1 +#~ SONO_TIMEOUT = 60 + +KUBERNETES_SERVICE_HOST=127.0.0.1 +KUBERNETES_SERVICE_PORT=34743 + + +############################################################################### +## Helpers: ## +############################################################################### + +ifeq ($(IMAGE_VERSION_TAG),) + export TAG=dev +else + export TAG=${IMAGE_VERSION_TAG} +endif + +SONOBUOY_IMAGE = "${IMAGE_REGISTRY}/${IMAGE_NAME}:${TAG}" + +container-init: + @echo "" + @echo "[ContainerImageName] ${SONOBUOY_IMAGE}" + @echo "[SonobuoyPluginFile] ${PLUGIN_FILE}" + @echo "" + + +kind-init: + @echo "" + @echo "[KindCluster] ${KIND_CLUSTER}" + @echo "" + + +############################################################################### +## For develpoment usage: ## +############################################################################### + +dev-prerequests: + @echo "[check-test-setup]" + @kind version + @docker version + @sonobuoy version --short + @go version + + +dev-setup: kind-init + kind create cluster --name ${KIND_CLUSTER} --config kind-config.yaml + + +dev-build: container-init + @echo "[build]" + DOCKER_BUILDKIT=1 docker build . -f ${DOCKERFILE} -t ${SONOBUOY_IMAGE} + kind load docker-image --name ${KIND_CLUSTER} ${SONOBUOY_IMAGE} + + +dev-go: + @echo "[go]" + @echo "[KubernetesService] ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" + @rm -rf ./build || true + @mkdir ./build + go test -c -o ./build ./... +# go test -c -o ./build ./... --args --skip-labels="type=pod-list" +# go tool test2json ./build -test.v + + +dev-run: + @echo "[run-test]" + @echo "sonobuoy run --plugin ${PLUGIN_FILE} --wait=${SONO_WAIT} --timeout=${SONO_TIMEOUT}" +#~ @sonobuoy run --plugin ${PLUGIN_FILE} --wait=${SONO_WAIT} --timeout=${SONO_TIMEOUT} + @sonobuoy run --plugin ${PLUGIN_FILE} + @sonobuoy status + + +dev-result: + @echo "[result]" + #outfile=$(sonobuoy retrieve) && mkdir results && tar -xf ${outfile} -C results + sonobuoy retrieve + sonobuoy results *.tar.gz + mkdir results + tar -xf *.tar.gz -C results + + +dev-clean: + @echo "[clean]" + @sonobuoy delete --all --wait || true + @sonobuoy status || true + @rm -rf *.tar.gz || true + @rm -rf results || true + + +dev-purge: kind-init dev-clean + @echo "[purge]" + kind delete cluster --name ${KIND_CLUSTER} || true + docker rmi ${SONOBUOY_IMAGE} || true + + +dev-rerun: dev-clean dev-build dev-run + @echo "[rerun] Waiting for tests to complete..." + # Poll sonobuoy status until it shows "complete" + while true; do \ + status_output=$$(sonobuoy status); \ + echo "$$status_output"; \ + if echo "$$status_output" | grep -q "complete"; then \ + echo "Tests completed."; \ + break; \ + fi; \ + sleep 5; \ + done + @echo "[rerun] Waiting an additional 10 seconds to ensure results are ready..." + sleep 30 + $(MAKE) dev-result + @echo "[Displaying results...]" + cat results/plugins/scsconformance/sonobuoy_results.yaml + cat results/plugins/scsconformance/results/global/out.json + +PHONY: dev-prerequests dev-build dev-run dev-result dev-clean dev-clean dev-purge diff --git a/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml b/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml new file mode 100644 index 000000000..fc1e05269 --- /dev/null +++ b/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml @@ -0,0 +1,19 @@ +# kind-config.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +kubeadmConfigPatches: +- | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration + apiServer: + extraArgs: + enable-admission-plugins: EventRateLimit + admission-control-config-file: /etc/kubernetes/admission-control-config.yaml + feature-gates": APIPriorityAndFairness=true +nodes: +- role: control-plane + extraMounts: + - hostPath: ./event-ratelimit-config.yaml + containerPath: /etc/kubernetes/admission-control-config.yaml +- role: worker +- role: worker \ No newline at end of file diff --git a/Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml b/Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml new file mode 100644 index 000000000..7dc0513a7 --- /dev/null +++ b/Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml @@ -0,0 +1,8 @@ +# event-ratelimit-config.yaml +kind: Configuration +apiVersion: eventratelimit.admission.k8s.io/v1alpha1 +limits: +- burst: 20000 + qps: 5000 + type: Server + diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go index 7d0b011e7..5976423c6 100644 --- a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go +++ b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go @@ -10,14 +10,16 @@ import ( "testing" "time" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ) // getClusterSize estimates the cluster size by counting the number of nodes. func getClusterSize(clientset *kubernetes.Clientset) int { - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{}) + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { log.Fatalf("Failed to list nodes: %v", err) } @@ -35,7 +37,7 @@ func runConcurrentRequests(clientset *kubernetes.Clientset, maxRequestsInflight defer wg.Done() ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) defer cancel() - _, err := clientset.CoreV1().Pods("").List(ctx, v1.ListOptions{}) + _, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) if err != nil { errChan <- err } @@ -62,57 +64,259 @@ func testPositiveCase(t *testing.T, clientset *kubernetes.Clientset, maxRequests // testNegativeCase handles the negative scenario where requests should be throttled. func testNegativeCase(t *testing.T, clientset *kubernetes.Clientset, maxRequestsInflight int) { - fmt.Println("Running Negative Test Case") - // Increase the load significantly above the maxRequestsInflight to trigger rate limiting - overloadRequests := maxRequestsInflight * 2 - errors := runConcurrentRequests(clientset, overloadRequests) - - // Expect at least some errors due to rate limiting - if errors == 0 { - t.Errorf("Test failed: expected rate limit errors, but all requests succeeded.") - } else { - fmt.Println("Negative test case passed as expected: rate limit exceeded.") - } + fmt.Println("Running Negative Test Case") + // Increase the load significantly above the maxRequestsInflight to trigger rate limiting + overloadRequests := maxRequestsInflight * 2 + errors := runConcurrentRequests(clientset, overloadRequests) + + // Expect at least some errors due to rate limiting + if errors == 0 { + t.Errorf("Test failed: expected rate limit errors, but all requests succeeded.") + } else { + fmt.Println("Negative test case passed as expected: rate limit exceeded.") + } } // Test_scs_maxRequestInflight is the main entry point that runs both positive and negative test cases. func Test_scs_0215_maxRequestInflight(t *testing.T) { - // Load in-cluster configuration - config, err := rest.InClusterConfig() - if err != nil { - log.Fatalf("Failed to load in-cluster config: %v", err) - } - - // Adjust client rate limits - config.QPS = 50 // Queries Per Second - config.Burst = 100 // Allowed burst (concurrent requests above QPS) - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatalf("Failed to create Kubernetes client: %v", err) - } - + // Load in-cluster configuration + config, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("Failed to load in-cluster config: %v", err) + } - // Get cluster size (number of nodes) - clusterSize := getClusterSize(clientset) - fmt.Printf("Detected cluster size: %d nodes\n", clusterSize) + // Adjust client rate limits + config.QPS = 10000 // Matches server-side QPS + config.Burst = 40000 // Matches server-side Burst - // Determine maxRequestsInflight based on cluster size and environment variable - maxRequestsInflightStr := os.Getenv("MAX_REQUESTS_INFLIGHT") - maxRequestsInflight, err := strconv.Atoi(maxRequestsInflightStr) - if err != nil || maxRequestsInflight <= 0 { - maxRequestsInflight = clusterSize * 250 // Example scaling logic: 100 requests per node - } + // Create the clientset from the config + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create Kubernetes clientset: %v", err) + } + + // Increase timeout to allow more time for requests to complete + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + // Example of using the clientset to list pods + _, err = clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + if err != nil { + if isRateLimitError(err) { + log.Printf("Rate limit error: %v", err) + } else { + log.Printf("Unexpected error: %v", err) // Log unexpected errors with details + } + } + + // Get cluster size (number of nodes) + clusterSize := getClusterSize(clientset) + fmt.Printf("Detected cluster size: %d nodes\n", clusterSize) + + // Determine maxRequestsInflight based on cluster size and environment variable + maxRequestsInflightStr := os.Getenv("MAX_REQUESTS_INFLIGHT") + maxRequestsInflight, err := strconv.Atoi(maxRequestsInflightStr) + if err != nil || maxRequestsInflight <= 0 { + maxRequestsInflight = clusterSize * 250 // Example scaling logic: 100 requests per node + } + + fmt.Printf("Using maxRequestsInflight = %d\n", maxRequestsInflight) + + // Run the positive test case + t.Run("Positive Test Case", func(t *testing.T) { + testPositiveCase(t, clientset, maxRequestsInflight) + }) + + // Run the negative test case + t.Run("Negative Test Case", func(t *testing.T) { + testNegativeCase(t, clientset, maxRequestsInflight) + }) +} + +// Main test function for max-mutating-requests-inflight +func Test_scs_maxMutatingRequestsInflight(t *testing.T) { + // Load in-cluster configuration + config, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("Failed to load in-cluster config: %v", err) + } + + // Set higher QPS and burst limits to avoid client-side throttling + config.QPS = 100 + config.Burst = 200 + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create Kubernetes client: %v", err) + } + + clusterSize := detectClusterSize() // Detects the cluster size + maxMutatingRequestsInflight := calculateMaxMutatingRequestsInflight(clusterSize) + fmt.Printf("Detected cluster size: %d nodes\n", clusterSize) + fmt.Printf("Using maxMutatingRequestsInflight = %d\n", maxMutatingRequestsInflight) + + // Positive Test Case: Requests within the allowed limit + t.Run("Positive_Test_Case", func(t *testing.T) { + fmt.Println("Running Positive Test Case") + err := runMutatingTest(clientset, maxMutatingRequestsInflight) // Pass clientset here + if err != nil { + t.Fatalf("Test failed: encountered unexpected errors when requests were expected to succeed: %v", err) + } + fmt.Println("Positive test case passed successfully!") + }) + + // Negative Test Case: Exceeding the allowed limit + t.Run("Negative_Test_Case", func(t *testing.T) { + fmt.Println("Running Negative Test Case") + err := runMutatingTest(clientset, maxMutatingRequestsInflight + 10) // Pass clientset here and exceed limit + if err != nil && isRateLimitError(err) { + fmt.Println("Negative test case passed as expected: rate limit exceeded.") + } else { + t.Fatalf("Test failed: expected rate limit errors, but requests succeeded or another error occurred: %v", err) + } + }) +} - fmt.Printf("Using maxRequestsInflight = %d\n", maxRequestsInflight) - // Run the positive test case - t.Run("Positive Test Case", func(t *testing.T) { - testPositiveCase(t, clientset, maxRequestsInflight) - }) +// Function to detect the size of the cluster (stubbed, adjust as needed) +func detectClusterSize() int { + // Logic to detect cluster size (for example using kubectl) + return 1 // Default for single-node kind cluster +} + +// Function to calculate max-mutating-requests-inflight based on cluster size +func calculateMaxMutatingRequestsInflight(clusterSize int) int { + // Adjust this formula based on your requirements + return 100 * clusterSize // Example: 100 mutating requests per node +} + +// Function to simulate sending mutating requests up to the given limit +func runMutatingTest(clientset *kubernetes.Clientset, limit int) error { + var wg sync.WaitGroup + errChan := make(chan error, limit) + + for i := 0; i < limit; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.TODO(), 20*time.Second) + defer cancel() + + // Create a unique Pod name + podName := fmt.Sprintf("test-pod-%d", i) + + // Create a Pod + _, err := clientset.CoreV1().Pods("default").Create(ctx, &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Image: "busybox", + }, + }, + }, + }, metav1.CreateOptions{}) + + if err != nil { + if isRateLimitError(err) { + errChan <- fmt.Errorf("rate limit reached") + } else { + errChan <- fmt.Errorf("error creating pod: %v", err) + } + return + } + + // Clean up by deleting the Pod + err = clientset.CoreV1().Pods("default").Delete(ctx, podName, metav1.DeleteOptions{}) + if err != nil { + if isRateLimitError(err) { + errChan <- fmt.Errorf("rate limit reached") + } else { + errChan <- fmt.Errorf("error deleting pod: %v", err) + } + return + } + }(i) + } + + wg.Wait() + close(errChan) + + var rateLimitErrors, otherErrors int + for err := range errChan { + if err.Error() == "rate limit reached" { + rateLimitErrors++ + } else { + otherErrors++ + } + } + + if otherErrors > 0 { + return fmt.Errorf("encountered %d unexpected errors", otherErrors) + } - // Run the negative test case - t.Run("Negative Test Case", func(t *testing.T) { - testNegativeCase(t, clientset, maxRequestsInflight) - }) + if rateLimitErrors > 0 { + fmt.Printf("Rate limit errors encountered: %d\n", rateLimitErrors) + } + + return nil +} + +// Function to determine if an error is related to rate limiting +func isRateLimitError(err error) bool { + if err == nil { + return false + } + return err.Error() == "TooManyRequests" || err.Error() == "429" +} + +// Main test function for min-request-timeout +func Test_scs_minRequestTimeout(t *testing.T) { + // Load in-cluster configuration + config, err := rest.InClusterConfig() + if err != nil { + log.Fatalf("Failed to load in-cluster config: %v", err) + } + + // Set QPS and Burst to higher values to avoid throttling + config.QPS = 100 + config.Burst = 200 + + // Create a Kubernetes client + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create Kubernetes client: %v", err) + } + + // Test case: min-request-timeout enforced (timeout set to 5 seconds) + t.Run("Test_minRequestTimeout", func(t *testing.T) { + minRequestTimeout := 5 * time.Second + fmt.Printf("Testing with min-request-timeout = %v\n", minRequestTimeout) + + ctx, cancel := context.WithTimeout(context.Background(), minRequestTimeout) + defer cancel() + + // Send a request to the Kubernetes API (List Pods in a namespace) + _, err := clientset.CoreV1().Pods("default").List(ctx, metav1.ListOptions{}) + + // Check if the request failed due to timeout + if err != nil && isTimeoutError(err) { + fmt.Printf("Request failed as expected due to timeout: %v\n", err) + } else if err != nil { + t.Fatalf("Test failed: unexpected error occurred: %v\n", err) + } else { + t.Fatalf("Test failed: request succeeded but was expected to timeout") + } + }) } + +// Helper function to check if an error is a timeout error +func isTimeoutError(err error) bool { + if err == nil { + return false + } + return err.Error() == "context deadline exceeded" +} \ No newline at end of file From 7013a9db61f1df7558447817036354ef6d2e5379 Mon Sep 17 00:00:00 2001 From: cah-patrickthiem Date: Wed, 16 Oct 2024 11:42:16 +0200 Subject: [PATCH 3/6] add results path to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 2b83a0983..4abd980fc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ .sandbox .DS_Store node_modules +<<<<<<< HEAD Tests/kaas/results/ +======= +>>>>>>> 4e8fc4d (add results path to .gitignore) Tests/kaas/kaas-sonobuoy-tests/results/ *.tar.gz From 9f4ec8526316980ca78dc2bc3a45b98180ae3fd3 Mon Sep 17 00:00:00 2001 From: cah-patrickthiem Date: Mon, 11 Nov 2024 09:43:33 +0100 Subject: [PATCH 4/6] Added remaining robustness tests. --- .../Makefile | 132 --- .../kind-config.yaml | 19 - Tests/kaas/kaas-sonobuoy-tests/Makefile | 2 +- .../kaas/kaas-sonobuoy-tests/kind_config.yaml | 4 +- .../scs_0215_v1_robustness_features_test.go | 826 ++++++++++++++++++ .../scs_0215_v1_robustness_features_test.go | 322 ------- 6 files changed, 828 insertions(+), 477 deletions(-) delete mode 100644 Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile delete mode 100644 Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml create mode 100644 Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go delete mode 100644 Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go diff --git a/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile b/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile deleted file mode 100644 index 2892164bb..000000000 --- a/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/Makefile +++ /dev/null @@ -1,132 +0,0 @@ -# Makefile -# This makefile is for development purpose - -SHELL = /bin/bash -#SED ?= sed - -DOCKERFILE="Dockerfile" -IMAGE_REGISTRY="ghcr.io/sovereigncloudstack/standards" -IMAGE_NAME="scsconformance" -#IMAGE_VERSION_TAG ="v0.1.2" - -KIND_CLUSTER ="testcluster" - -#PLUGIN_NAME="k8s-default-storage-class-plugin-go" -PLUGIN_NAME="plugin" -PLUGIN_FILE="${PLUGIN_NAME}.yaml" - -#~ SONO_WAIT = 1 -#~ SONO_TIMEOUT = 60 - -KUBERNETES_SERVICE_HOST=127.0.0.1 -KUBERNETES_SERVICE_PORT=34743 - - -############################################################################### -## Helpers: ## -############################################################################### - -ifeq ($(IMAGE_VERSION_TAG),) - export TAG=dev -else - export TAG=${IMAGE_VERSION_TAG} -endif - -SONOBUOY_IMAGE = "${IMAGE_REGISTRY}/${IMAGE_NAME}:${TAG}" - -container-init: - @echo "" - @echo "[ContainerImageName] ${SONOBUOY_IMAGE}" - @echo "[SonobuoyPluginFile] ${PLUGIN_FILE}" - @echo "" - - -kind-init: - @echo "" - @echo "[KindCluster] ${KIND_CLUSTER}" - @echo "" - - -############################################################################### -## For develpoment usage: ## -############################################################################### - -dev-prerequests: - @echo "[check-test-setup]" - @kind version - @docker version - @sonobuoy version --short - @go version - - -dev-setup: kind-init - kind create cluster --name ${KIND_CLUSTER} --config kind-config.yaml - - -dev-build: container-init - @echo "[build]" - DOCKER_BUILDKIT=1 docker build . -f ${DOCKERFILE} -t ${SONOBUOY_IMAGE} - kind load docker-image --name ${KIND_CLUSTER} ${SONOBUOY_IMAGE} - - -dev-go: - @echo "[go]" - @echo "[KubernetesService] ${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}" - @rm -rf ./build || true - @mkdir ./build - go test -c -o ./build ./... -# go test -c -o ./build ./... --args --skip-labels="type=pod-list" -# go tool test2json ./build -test.v - - -dev-run: - @echo "[run-test]" - @echo "sonobuoy run --plugin ${PLUGIN_FILE} --wait=${SONO_WAIT} --timeout=${SONO_TIMEOUT}" -#~ @sonobuoy run --plugin ${PLUGIN_FILE} --wait=${SONO_WAIT} --timeout=${SONO_TIMEOUT} - @sonobuoy run --plugin ${PLUGIN_FILE} - @sonobuoy status - - -dev-result: - @echo "[result]" - #outfile=$(sonobuoy retrieve) && mkdir results && tar -xf ${outfile} -C results - sonobuoy retrieve - sonobuoy results *.tar.gz - mkdir results - tar -xf *.tar.gz -C results - - -dev-clean: - @echo "[clean]" - @sonobuoy delete --all --wait || true - @sonobuoy status || true - @rm -rf *.tar.gz || true - @rm -rf results || true - - -dev-purge: kind-init dev-clean - @echo "[purge]" - kind delete cluster --name ${KIND_CLUSTER} || true - docker rmi ${SONOBUOY_IMAGE} || true - - -dev-rerun: dev-clean dev-build dev-run - @echo "[rerun] Waiting for tests to complete..." - # Poll sonobuoy status until it shows "complete" - while true; do \ - status_output=$$(sonobuoy status); \ - echo "$$status_output"; \ - if echo "$$status_output" | grep -q "complete"; then \ - echo "Tests completed."; \ - break; \ - fi; \ - sleep 5; \ - done - @echo "[rerun] Waiting an additional 10 seconds to ensure results are ready..." - sleep 30 - $(MAKE) dev-result - @echo "[Displaying results...]" - cat results/plugins/scsconformance/sonobuoy_results.yaml - cat results/plugins/scsconformance/results/global/out.json - -PHONY: dev-prerequests dev-build dev-run dev-result dev-clean dev-clean dev-purge diff --git a/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml b/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml deleted file mode 100644 index fc1e05269..000000000 --- a/Tests/kaas/kaas-sonobuoy-go-example-e2e-framework/kind-config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# kind-config.yaml -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -kubeadmConfigPatches: -- | - apiVersion: kubeadm.k8s.io/v1beta2 - kind: ClusterConfiguration - apiServer: - extraArgs: - enable-admission-plugins: EventRateLimit - admission-control-config-file: /etc/kubernetes/admission-control-config.yaml - feature-gates": APIPriorityAndFairness=true -nodes: -- role: control-plane - extraMounts: - - hostPath: ./event-ratelimit-config.yaml - containerPath: /etc/kubernetes/admission-control-config.yaml -- role: worker -- role: worker \ No newline at end of file diff --git a/Tests/kaas/kaas-sonobuoy-tests/Makefile b/Tests/kaas/kaas-sonobuoy-tests/Makefile index dffc6d7a2..d84eae204 100644 --- a/Tests/kaas/kaas-sonobuoy-tests/Makefile +++ b/Tests/kaas/kaas-sonobuoy-tests/Makefile @@ -62,7 +62,7 @@ dev-setup: kind-init dev-build: container-init @echo "[Building image...]" - DOCKER_BUILDKIT=1 docker build . -f ${DOCKERFILE} -t ${SONOBUOY_IMAGE} + DOCKER_BUILDKIT=1 docker build . -f ${DOCKERFILE} -t ${SONOBUOY_IMAGE} --load kind load docker-image --name ${KIND_CLUSTER} ${SONOBUOY_IMAGE} diff --git a/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml b/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml index 947a9fa8a..371c74d4c 100644 --- a/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml +++ b/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml @@ -5,6 +5,4 @@ networking: apiServerPort: 6443 nodes: - role: control-plane -- role: worker -- role: worker -- role: worker +- role: worker \ No newline at end of file diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go new file mode 100644 index 000000000..e5d8022c4 --- /dev/null +++ b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go @@ -0,0 +1,826 @@ +package scs_k8s_tests + +import ( + "context" + "crypto/x509" + "encoding/pem" + "fmt" + "reflect" + "log" + "os" + "strconv" + "strings" + "sync" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// Configuration constants +const ( + defaultQPS = 100 + defaultBurst = 200 +) + +// ==================== Helper Functions ==================== + +// Cluster Information Helpers + +// getClusterSize returns the total number of nodes in the cluster +func getClusterSize(clientset *kubernetes.Clientset) int { + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + log.Fatalf("Failed to list nodes: %v", err) + } + return len(nodes.Items) +} + +// isKindCluster determines if we're running on a kind cluster by checking node labels +func isKindCluster(clientset *kubernetes.Clientset) bool { + nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return false + } + for _, node := range nodes.Items { + if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok { + return true + } + } + return false +} + +// Configuration and Environment Helpers + +// getEnvOrDefault retrieves an environment variable or returns a default value +func getEnvOrDefault(key string, defaultValue int) int { + if value, exists := os.LookupEnv(key); exists { + if intValue, err := strconv.Atoi(value); err == nil { + return intValue + } + } + return defaultValue +} + +// setupClientset creates a Kubernetes clientset with specified QPS and Burst +func setupClientset(qps, burst float32) (*kubernetes.Clientset, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to load in-cluster config: %v", err) + } + + config.QPS = qps + config.Burst = int(burst) + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes clientset: %v", err) + } + + return clientset, nil +} + +// Error Detection Helpers + +// isRateLimitError checks if an error is related to rate limiting +func isRateLimitError(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "429") || + strings.Contains(strings.ToLower(err.Error()), "too many requests") || + strings.Contains(strings.ToLower(err.Error()), "rate limit") +} + +// isTimeoutError checks if an error is related to timeouts +func isTimeoutError(err error) bool { + if err == nil { + return false + } + return err == context.DeadlineExceeded || + strings.Contains(err.Error(), "context deadline exceeded") || + strings.Contains(strings.ToLower(err.Error()), "timeout") +} + +// Certificate Handling Helper + +// parseCertificate decodes and parses an X.509 certificate from PEM format +func parseCertificate(certData []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(certData) + if block == nil { + return nil, fmt.Errorf("failed to decode PEM block") + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse certificate: %v", err) + } + + return cert, nil +} + +// runConcurrentRequests executes multiple concurrent API requests and counts errors +func runConcurrentRequests(clientset *kubernetes.Clientset, requestCount int) int { + var wg sync.WaitGroup + errChan := make(chan error, requestCount) + + for i := 0; i < requestCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) + defer cancel() + _, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) + if err != nil { + errChan <- err + } + }() + } + + wg.Wait() + close(errChan) + return len(errChan) +} + +// generateComplexFieldSelector creates a complex field selector for testing timeouts +func generateComplexFieldSelector() []string { + selectors := make([]string, 100) + for i := 0; i < 100; i++ { + selectors[i] = fmt.Sprintf("metadata.name!=%d", i) + } + return selectors +} + +// calculateMaxMutatingRequestsInflight determines the maximum number of mutating requests +func calculateMaxMutatingRequestsInflight(clusterSize int) int { + return getEnvOrDefault("MAX_MUTATING_REQUESTS_INFLIGHT", clusterSize*50) +} + +// hasEtcdBackup is a generic helper to check for etcd backup in various resources +func hasEtcdBackup[T any](items []T, nameCheck func(string) bool) bool { + for _, item := range items { + name := reflect.ValueOf(item).FieldByName("Name").String() + if nameCheck(name) { + return true + } + } + return false +} + +// findCertificateData looks for certificate data in secret data map +func findCertificateData(data map[string][]byte) []byte { + // First check for standard TLS certificate key + if certData, ok := data["tls.crt"]; ok { + return certData + } + + // Try alternative key names + for key, data := range data { + if strings.Contains(key, ".crt") || strings.Contains(key, "cert") { + return data + } + } + return nil +} + +// ==================== Test Cases ==================== + +// Test groups are organized by feature: +// 1. API Request Limits +// 2. Request Timeout +// 3. Event Rate Limiting +// 4. API Priority and Fairness +// 5. etcd Management +// 6. Certificate Management + +// -------------------- 1. API Request Limits -------------------- + +// Test_scs_0215_maxRequestInflight verifies the API server's max-requests-inflight setting +func Test_scs_0215_maxRequestInflight(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + clusterSize := getClusterSize(clientset) + maxRequestsInflight := getEnvOrDefault("MAX_REQUESTS_INFLIGHT", clusterSize*250) + + t.Logf("Detected cluster size: %d nodes", clusterSize) + t.Logf("Using maxRequestsInflight = %d", maxRequestsInflight) + + t.Run("Positive_Test_Case", func(t *testing.T) { + // Test with safe number of requests (25% of limit) + t.Log("Running Positive Test Case") + safeRequests := maxRequestsInflight / 4 + errors := runConcurrentRequests(clientset, safeRequests) + if errors > 0 { + t.Errorf("Test failed: encountered %d unexpected errors when requests were expected to succeed.", errors) + } else { + t.Log("Positive test case passed successfully!") + } + }) + + t.Run("Negative_Test_Case", func(t *testing.T) { + // Test with excessive requests (200% of limit) + t.Log("Running Negative Test Case") + overloadRequests := maxRequestsInflight * 2 + errors := runConcurrentRequests(clientset, overloadRequests) + + if errors == 0 { + t.Error("Test failed: expected rate limit errors, but all requests succeeded") + } else { + t.Log("Negative test case passed as expected: rate limit exceeded") + } + }) +} + +// Test_scs_0215_maxMutatingRequestsInflight verifies the API server's max-mutating-requests-inflight setting +func Test_scs_0215_maxMutatingRequestsInflight(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + clusterSize := getClusterSize(clientset) + maxMutatingRequestsInflight := calculateMaxMutatingRequestsInflight(clusterSize) + isKind := isKindCluster(clientset) + + t.Logf("Detected cluster size: %d nodes", clusterSize) + t.Logf("Using maxMutatingRequestsInflight = %d", maxMutatingRequestsInflight) + + t.Run("Positive_Test_Case", func(t *testing.T) { + t.Log("Running Positive Test Case") + err := runMutatingTest(clientset, maxMutatingRequestsInflight/2) + if err != nil { + t.Fatalf("Positive test failed: %v", err) + } + t.Log("Positive test case passed successfully!") + }) + + t.Run("Negative_Test_Case", func(t *testing.T) { + t.Log("Running Negative Test Case") + // Use higher QPS/burst for negative test case + highLoadClientset, _ := setupClientset(2000, 2000) + + var wg sync.WaitGroup + errorChan := make(chan error, 5) + + // Run multiple concurrent batches of requests + batchCount := 5 + if isKind { + batchCount = 2 // Use fewer batches for kind clusters + } + + for i := 0; i < batchCount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := runMutatingTest(highLoadClientset, maxMutatingRequestsInflight) + if err != nil { + errorChan <- err + } + }() + } + + wg.Wait() + close(errorChan) + + var gotRateLimit bool + for err := range errorChan { + if isRateLimitError(err) { + gotRateLimit = true + break + } + } + + if !gotRateLimit { + if isKind { + t.Log("No rate limit error in kind cluster - this is acceptable") + } else { + t.Error("Expected rate limit errors in production cluster, but got none") + } + } + }) +} + +// runMutatingTest performs concurrent mutating requests to test rate limiting +func runMutatingTest(clientset *kubernetes.Clientset, limit int) error { + var wg sync.WaitGroup + errChan := make(chan error, limit) + + for i := 0; i < limit; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) + defer cancel() + + _, err := clientset.CoreV1().ConfigMaps("default").Create(ctx, &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("test-cm-%d-", i), + }, + Data: map[string]string{ + "test": fmt.Sprintf("data-%d", i), + "load": strings.Repeat("x", 1000), // Add some data to increase request size + }, + }, metav1.CreateOptions{}) + + if err != nil { + errChan <- err + } + }(i) + } + + wg.Wait() + close(errChan) + + var lastError error + for err := range errChan { + if isRateLimitError(err) { + return err + } + lastError = err + } + + return lastError +} + +// -------------------- 2. Request Timeout -------------------- + +// Test_scs_0215_minRequestTimeout verifies the API server's min-request-timeout setting +func Test_scs_0215_minRequestTimeout(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + t.Run("Test_minRequestTimeout", func(t *testing.T) { + isKind := isKindCluster(clientset) + minRequestTimeout := time.Duration(getEnvOrDefault("MIN_REQUEST_TIMEOUT", 1)) * time.Second + t.Logf("Testing with min-request-timeout = %v", minRequestTimeout) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + startTime := time.Now() + _, err := clientset.CoreV1().Pods("default").List(ctx, metav1.ListOptions{ + FieldSelector: strings.Join(generateComplexFieldSelector(), ","), + }) + duration := time.Since(startTime) + + if isKind { + t.Logf("Running on kind cluster. Request completed in %v", duration) + } else { + if err == nil && duration < minRequestTimeout { + t.Errorf("Request completed faster than minimum timeout. Duration: %v, Expected minimum: %v", duration, minRequestTimeout) + } else { + t.Logf("Request completed as expected in %v", duration) + } + } + }) +} + +// -------------------- 3. Event Rate Limiting -------------------- + +// Test_scs_0215_eventRateLimit verifies the EventRateLimit admission controller configuration +func Test_scs_0215_eventRateLimit(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + isKind := isKindCluster(clientset) + if isKind { + t.Skip("Running on kind cluster - skipping EventRateLimit test as it's not required for development environments") + } + + t.Run("Check_EventRateLimit_Configuration", func(t *testing.T) { + // Check configurations in multiple locations + configLocations := []struct { + name string + namespace string + key string + }{ + {"admission-configuration", "kube-system", "eventratelimit.yaml"}, + {"kube-apiserver", "kube-system", "config.yaml"}, + } + + for _, loc := range configLocations { + config, err := clientset.CoreV1().ConfigMaps(loc.namespace).Get( + context.TODO(), + loc.name, + metav1.GetOptions{}, + ) + if err == nil { + if data, ok := config.Data[loc.key]; ok { + if strings.Contains(data, "eventratelimit.admission.k8s.io") { + t.Logf("Found EventRateLimit configuration in %s/%s", loc.namespace, loc.name) + return + } + } + } + } + + // Check for standalone configuration + configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.TODO(), metav1.ListOptions{}) + for _, cm := range configMaps.Items { + if strings.Contains(cm.Name, "event-rate-limit") { + t.Logf("Found standalone EventRateLimit configuration in ConfigMap: %s", cm.Name) + return + } + } + + t.Error("No EventRateLimit configuration found in production cluster") + }) +} + +// -------------------- 4. API Priority and Fairness -------------------- + +// Test_scs_0215_apiPriorityAndFairness verifies API Priority and Fairness (APF) configuration +func Test_scs_0215_apiPriorityAndFairness(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + isKind := isKindCluster(clientset) + if isKind { + t.Skip("Running on kind cluster - skipping APF test as it's configured differently in development environments") + } + + t.Run("Check_APF_Configuration", func(t *testing.T) { + // Multiple checks for APF configuration + checks := []struct { + name string + fn func() bool + }{ + {"API Server Config", func() bool { + config, err := clientset.CoreV1().ConfigMaps("kube-system").Get( + context.TODO(), + "kube-apiserver", + metav1.GetOptions{}, + ) + return err == nil && config.Data["config.yaml"] != "" && + strings.Contains(config.Data["config.yaml"], "enable-priority-and-fairness: true") + }}, + {"Command Line Flags", func() bool { + pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil || len(pods.Items) == 0 { + return false + } + for _, pod := range pods.Items { + for _, container := range pod.Spec.Containers { + for _, arg := range container.Command { + if strings.Contains(arg, "--enable-priority-and-fairness=true") { + return true + } + } + } + } + return false + }}, + {"API Resources", func() bool { + resources, err := clientset.Discovery().ServerResourcesForGroupVersion("flowcontrol.apiserver.k8s.io/v1beta3") + if err != nil || resources == nil { + return false + } + for _, r := range resources.APIResources { + if r.Name == "flowschemas" || r.Name == "prioritylevelconfigurations" { + return true + } + } + return false + }}, + } + + for _, check := range checks { + if check.fn() { + t.Logf("APF enabled via %s", check.name) + return + } + } + + t.Error("No API Priority and Fairness configuration found in production cluster") + }) +} + +// Test_scs_0215_rateLimitValues verifies recommended rate limit values +func Test_scs_0215_rateLimitValues(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + isKind := isKindCluster(clientset) + if isKind { + t.Skip("Running on kind cluster - skipping rate limit values test") + } + + t.Run("Check_Rate_Limit_Values", func(t *testing.T) { + // Define expected values + expectedValues := map[string]string{ + "qps": "5000", + "burst": "20000", + } + + // Check various possible configuration locations + configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.TODO(), metav1.ListOptions{}) + for _, cm := range configMaps.Items { + var config string + switch { + case strings.Contains(cm.Name, "event-rate-limit"): + config = cm.Data["config.yaml"] + case cm.Name == "admission-configuration": + config = cm.Data["eventratelimit.yaml"] + case cm.Name == "kube-apiserver": + config = cm.Data["config.yaml"] + } + + if config != "" { + allFound := true + for k, v := range expectedValues { + if !strings.Contains(config, fmt.Sprintf("%s: %s", k, v)) { + allFound = false + break + } + } + if allFound { + t.Log("Found recommended rate limit values") + return + } + } + } + + t.Error("Recommended rate limit values (qps: 5000, burst: 20000) not found") + }) +} + +// -------------------- 5. etcd Management -------------------- + +// Test_scs_0215_etcdCompaction verifies etcd compaction settings +func Test_scs_0215_etcdCompaction(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + isKind := isKindCluster(clientset) + if isKind { + t.Skip("Running on kind cluster - skipping etcd compaction test as it has different default settings") + } + + t.Run("Check_Etcd_Compaction_Settings", func(t *testing.T) { + // Try different label selectors for etcd pods + selectors := []string{ + "component=etcd", + "k8s-app=etcd", + "tier=control-plane,component=etcd", + } + + var etcdPods []v1.Pod + for _, selector := range selectors { + pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ + LabelSelector: selector, + }) + if err == nil && len(pods.Items) > 0 { + etcdPods = pods.Items + t.Logf("Found etcd pods using selector: %s", selector) + break + } + } + + if len(etcdPods) == 0 { + t.Skip("No etcd pods found - they might be running outside the cluster") + return + } + + // Expected settings + requiredSettings := map[string]string{ + "auto-compaction-mode": "periodic", + "auto-compaction-retention": "8h", + } + + // Check each etcd pod + for _, pod := range etcdPods { + t.Logf("Checking etcd pod: %s", pod.Name) + + // Check ConfigMap first + if cm, err := clientset.CoreV1().ConfigMaps("kube-system").Get( + context.TODO(), + "etcd-config", + metav1.GetOptions{}, + ); err == nil { + if config, ok := cm.Data["etcd.conf.yaml"]; ok { + allFound := true + for setting, value := range requiredSettings { + if !strings.Contains(config, fmt.Sprintf("%s: %s", setting, value)) { + allFound = false + break + } + } + if allFound { + t.Log("Found correct etcd compaction settings in ConfigMap") + return + } + } + } + + // Check command line arguments + for _, container := range pod.Spec.Containers { + foundSettings := make(map[string]bool) + for _, arg := range container.Command { + for setting, value := range requiredSettings { + if strings.Contains(arg, fmt.Sprintf("--%s=%s", setting, value)) { + foundSettings[setting] = true + } + } + } + + if len(foundSettings) == len(requiredSettings) { + t.Log("Found correct etcd compaction settings in command line arguments") + return + } + } + } + + t.Error("Required etcd compaction settings not found") + }) +} + +// Test_scs_0215_etcdBackup verifies etcd backup configuration +func Test_scs_0215_etcdBackup(t *testing.T) { + clientset, err := setupClientset(defaultQPS, defaultBurst) + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + isKind := isKindCluster(clientset) + if isKind { + t.Skip("Running on kind cluster - skipping etcd backup test as it's typically handled differently in development environments") + } + + t.Run("Check_Etcd_Backup_Configuration", func(t *testing.T) { + // Define backup solution checks + backupChecks := []struct { + name string + fn func() bool + }{ + {"CronJob", func() bool { + cronJobs, err := clientset.BatchV1().CronJobs("").List(context.TODO(), metav1.ListOptions{}) + return err == nil && hasEtcdBackup(cronJobs.Items, func(name string) bool { + return strings.Contains(strings.ToLower(name), "etcd") && + strings.Contains(strings.ToLower(name), "backup") + }) + }}, + {"Deployment", func() bool { + deployments, err := clientset.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{}) + return err == nil && hasEtcdBackup(deployments.Items, func(name string) bool { + return strings.Contains(strings.ToLower(name), "etcd") && + strings.Contains(strings.ToLower(name), "backup") + }) + }}, + {"DaemonSet", func() bool { + daemonSets, err := clientset.AppsV1().DaemonSets("").List(context.TODO(), metav1.ListOptions{}) + return err == nil && hasEtcdBackup(daemonSets.Items, func(name string) bool { + return strings.Contains(strings.ToLower(name), "etcd") && + strings.Contains(strings.ToLower(name), "backup") + }) + }}, + {"Dedicated Pods", func() bool { + pods, err := clientset.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{}) + return err == nil && hasEtcdBackup(pods.Items, func(name string) bool { + return strings.Contains(strings.ToLower(name), "etcd") && + strings.Contains(strings.ToLower(name), "backup") + }) + }}, + } + + for _, check := range backupChecks { + if check.fn() { + t.Logf("Found etcd backup solution: %s", check.name) + return + } + } + + t.Error("No etcd backup solution found. Required: at least weekly backups through CronJob, Deployment, DaemonSet, or dedicated Pods") + }) +} + +// -------------------- 6. Certificate Management -------------------- + +// Test_scs_0215_certificateRotation verifies certificate rotation configuration +func Test_scs_0215_certificateRotation(t *testing.T) { + clientset, err := setupClientset(100, 200) // Using default QPS and Burst + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + isKind := isKindCluster(clientset) + if isKind { + t.Skip("Running on kind cluster - skipping certificate rotation test") + } + + t.Run("Check_Certificate_Expiration", func(t *testing.T) { + secrets, err := clientset.CoreV1().Secrets("kube-system").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("Failed to list secrets: %v", err) + } + + oneYearFromNow := time.Now().AddDate(1, 0, 0) + certsChecked := 0 + + for _, secret := range secrets.Items { + // Check for TLS secrets and certificate-related secrets + if secret.Type == v1.SecretTypeTLS || + strings.Contains(secret.Name, "cert") || + strings.Contains(secret.Name, "certificate") { + + certData := findCertificateData(secret.Data) + if certData == nil { + continue + } + + cert, err := parseCertificate(certData) + if err != nil { + t.Logf("Failed to parse certificate from secret %s: %v", secret.Name, err) + continue + } + + certsChecked++ + t.Logf("Checking certificate in secret: %s, expires: %v", secret.Name, cert.NotAfter) + + if cert.NotAfter.Before(oneYearFromNow) { + t.Errorf("Certificate in secret %s expires in less than a year (expires: %v)", + secret.Name, cert.NotAfter) + } + } + } + + if certsChecked == 0 { + t.Error("No certificates found to check") + } else { + t.Logf("Checked %d certificates", certsChecked) + } + }) + + t.Run("Check_Certificate_Controller", func(t *testing.T) { + // Check for cert-manager deployment + deployments, err := clientset.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{}) + if err != nil { + t.Fatalf("Failed to list deployments: %v", err) + } + + // Look for various certificate controller implementations + certControllers := []string{ + "cert-manager", + "certificate-controller", + "cert-controller", + } + + found := false + for _, deployment := range deployments.Items { + for _, controller := range certControllers { + if strings.Contains(strings.ToLower(deployment.Name), controller) { + if deployment.Status.ReadyReplicas > 0 { + found = true + t.Logf("Found active certificate controller: %s", deployment.Name) + break + } + } + } + if found { + break + } + } + + if !found { + // Check for built-in controller in kube-controller-manager + pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ + LabelSelector: "component=kube-controller-manager", + }) + if err == nil && len(pods.Items) > 0 { + for _, pod := range pods.Items { + for _, container := range pod.Spec.Containers { + for _, arg := range container.Command { + if strings.Contains(arg, "--controllers=*") || + strings.Contains(arg, "--controllers=certificate") { + found = true + t.Log("Found built-in certificate controller in kube-controller-manager") + break + } + } + } + } + } + } + + if !found { + t.Error("No certificate controller found") + } + }) +} \ No newline at end of file diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go deleted file mode 100644 index 5976423c6..000000000 --- a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_tests/scs_0215_v1_robustness_features_test.go +++ /dev/null @@ -1,322 +0,0 @@ -package scs_k8s_tests - -import ( - "context" - "fmt" - "log" - "os" - "strconv" - "sync" - "testing" - "time" - - v1 "k8s.io/api/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - -) - -// getClusterSize estimates the cluster size by counting the number of nodes. -func getClusterSize(clientset *kubernetes.Clientset) int { - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - log.Fatalf("Failed to list nodes: %v", err) - } - return len(nodes.Items) -} - -// runConcurrentRequests sends concurrent API requests and returns the number of errors encountered. -func runConcurrentRequests(clientset *kubernetes.Clientset, maxRequestsInflight int) int { - var wg sync.WaitGroup - errChan := make(chan error, maxRequestsInflight) - - for i := 0; i < maxRequestsInflight; i++ { - wg.Add(1) - go func() { - defer wg.Done() - ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) - defer cancel() - _, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) - if err != nil { - errChan <- err - } - }() - } - - wg.Wait() - close(errChan) - return len(errChan) // Return the number of errors encountered -} - -// testPositiveCase handles the positive scenario where requests should succeed. -func testPositiveCase(t *testing.T, clientset *kubernetes.Clientset, maxRequestsInflight int) { - fmt.Println("Running Positive Test Case") - // Reduce the load even further for the positive test (e.g., 25% of maxRequestsInflight) - safeRequests := maxRequestsInflight / 4 - errors := runConcurrentRequests(clientset, safeRequests) - if errors > 0 { - t.Errorf("Test failed: encountered %d unexpected errors when requests were expected to succeed.", errors) - } else { - fmt.Println("Positive test case passed successfully!") - } -} - -// testNegativeCase handles the negative scenario where requests should be throttled. -func testNegativeCase(t *testing.T, clientset *kubernetes.Clientset, maxRequestsInflight int) { - fmt.Println("Running Negative Test Case") - // Increase the load significantly above the maxRequestsInflight to trigger rate limiting - overloadRequests := maxRequestsInflight * 2 - errors := runConcurrentRequests(clientset, overloadRequests) - - // Expect at least some errors due to rate limiting - if errors == 0 { - t.Errorf("Test failed: expected rate limit errors, but all requests succeeded.") - } else { - fmt.Println("Negative test case passed as expected: rate limit exceeded.") - } -} - -// Test_scs_maxRequestInflight is the main entry point that runs both positive and negative test cases. -func Test_scs_0215_maxRequestInflight(t *testing.T) { - // Load in-cluster configuration - config, err := rest.InClusterConfig() - if err != nil { - log.Fatalf("Failed to load in-cluster config: %v", err) - } - - // Adjust client rate limits - config.QPS = 10000 // Matches server-side QPS - config.Burst = 40000 // Matches server-side Burst - - // Create the clientset from the config - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatalf("Failed to create Kubernetes clientset: %v", err) - } - - // Increase timeout to allow more time for requests to complete - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - // Example of using the clientset to list pods - _, err = clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) - if err != nil { - if isRateLimitError(err) { - log.Printf("Rate limit error: %v", err) - } else { - log.Printf("Unexpected error: %v", err) // Log unexpected errors with details - } - } - - // Get cluster size (number of nodes) - clusterSize := getClusterSize(clientset) - fmt.Printf("Detected cluster size: %d nodes\n", clusterSize) - - // Determine maxRequestsInflight based on cluster size and environment variable - maxRequestsInflightStr := os.Getenv("MAX_REQUESTS_INFLIGHT") - maxRequestsInflight, err := strconv.Atoi(maxRequestsInflightStr) - if err != nil || maxRequestsInflight <= 0 { - maxRequestsInflight = clusterSize * 250 // Example scaling logic: 100 requests per node - } - - fmt.Printf("Using maxRequestsInflight = %d\n", maxRequestsInflight) - - // Run the positive test case - t.Run("Positive Test Case", func(t *testing.T) { - testPositiveCase(t, clientset, maxRequestsInflight) - }) - - // Run the negative test case - t.Run("Negative Test Case", func(t *testing.T) { - testNegativeCase(t, clientset, maxRequestsInflight) - }) -} - -// Main test function for max-mutating-requests-inflight -func Test_scs_maxMutatingRequestsInflight(t *testing.T) { - // Load in-cluster configuration - config, err := rest.InClusterConfig() - if err != nil { - log.Fatalf("Failed to load in-cluster config: %v", err) - } - - // Set higher QPS and burst limits to avoid client-side throttling - config.QPS = 100 - config.Burst = 200 - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatalf("Failed to create Kubernetes client: %v", err) - } - - clusterSize := detectClusterSize() // Detects the cluster size - maxMutatingRequestsInflight := calculateMaxMutatingRequestsInflight(clusterSize) - fmt.Printf("Detected cluster size: %d nodes\n", clusterSize) - fmt.Printf("Using maxMutatingRequestsInflight = %d\n", maxMutatingRequestsInflight) - - // Positive Test Case: Requests within the allowed limit - t.Run("Positive_Test_Case", func(t *testing.T) { - fmt.Println("Running Positive Test Case") - err := runMutatingTest(clientset, maxMutatingRequestsInflight) // Pass clientset here - if err != nil { - t.Fatalf("Test failed: encountered unexpected errors when requests were expected to succeed: %v", err) - } - fmt.Println("Positive test case passed successfully!") - }) - - // Negative Test Case: Exceeding the allowed limit - t.Run("Negative_Test_Case", func(t *testing.T) { - fmt.Println("Running Negative Test Case") - err := runMutatingTest(clientset, maxMutatingRequestsInflight + 10) // Pass clientset here and exceed limit - if err != nil && isRateLimitError(err) { - fmt.Println("Negative test case passed as expected: rate limit exceeded.") - } else { - t.Fatalf("Test failed: expected rate limit errors, but requests succeeded or another error occurred: %v", err) - } - }) -} - - -// Function to detect the size of the cluster (stubbed, adjust as needed) -func detectClusterSize() int { - // Logic to detect cluster size (for example using kubectl) - return 1 // Default for single-node kind cluster -} - -// Function to calculate max-mutating-requests-inflight based on cluster size -func calculateMaxMutatingRequestsInflight(clusterSize int) int { - // Adjust this formula based on your requirements - return 100 * clusterSize // Example: 100 mutating requests per node -} - -// Function to simulate sending mutating requests up to the given limit -func runMutatingTest(clientset *kubernetes.Clientset, limit int) error { - var wg sync.WaitGroup - errChan := make(chan error, limit) - - for i := 0; i < limit; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - ctx, cancel := context.WithTimeout(context.TODO(), 20*time.Second) - defer cancel() - - // Create a unique Pod name - podName := fmt.Sprintf("test-pod-%d", i) - - // Create a Pod - _, err := clientset.CoreV1().Pods("default").Create(ctx, &v1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - }, - Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: "test-container", - Image: "busybox", - }, - }, - }, - }, metav1.CreateOptions{}) - - if err != nil { - if isRateLimitError(err) { - errChan <- fmt.Errorf("rate limit reached") - } else { - errChan <- fmt.Errorf("error creating pod: %v", err) - } - return - } - - // Clean up by deleting the Pod - err = clientset.CoreV1().Pods("default").Delete(ctx, podName, metav1.DeleteOptions{}) - if err != nil { - if isRateLimitError(err) { - errChan <- fmt.Errorf("rate limit reached") - } else { - errChan <- fmt.Errorf("error deleting pod: %v", err) - } - return - } - }(i) - } - - wg.Wait() - close(errChan) - - var rateLimitErrors, otherErrors int - for err := range errChan { - if err.Error() == "rate limit reached" { - rateLimitErrors++ - } else { - otherErrors++ - } - } - - if otherErrors > 0 { - return fmt.Errorf("encountered %d unexpected errors", otherErrors) - } - - if rateLimitErrors > 0 { - fmt.Printf("Rate limit errors encountered: %d\n", rateLimitErrors) - } - - return nil -} - -// Function to determine if an error is related to rate limiting -func isRateLimitError(err error) bool { - if err == nil { - return false - } - return err.Error() == "TooManyRequests" || err.Error() == "429" -} - -// Main test function for min-request-timeout -func Test_scs_minRequestTimeout(t *testing.T) { - // Load in-cluster configuration - config, err := rest.InClusterConfig() - if err != nil { - log.Fatalf("Failed to load in-cluster config: %v", err) - } - - // Set QPS and Burst to higher values to avoid throttling - config.QPS = 100 - config.Burst = 200 - - // Create a Kubernetes client - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - log.Fatalf("Failed to create Kubernetes client: %v", err) - } - - // Test case: min-request-timeout enforced (timeout set to 5 seconds) - t.Run("Test_minRequestTimeout", func(t *testing.T) { - minRequestTimeout := 5 * time.Second - fmt.Printf("Testing with min-request-timeout = %v\n", minRequestTimeout) - - ctx, cancel := context.WithTimeout(context.Background(), minRequestTimeout) - defer cancel() - - // Send a request to the Kubernetes API (List Pods in a namespace) - _, err := clientset.CoreV1().Pods("default").List(ctx, metav1.ListOptions{}) - - // Check if the request failed due to timeout - if err != nil && isTimeoutError(err) { - fmt.Printf("Request failed as expected due to timeout: %v\n", err) - } else if err != nil { - t.Fatalf("Test failed: unexpected error occurred: %v\n", err) - } else { - t.Fatalf("Test failed: request succeeded but was expected to timeout") - } - }) -} - -// Helper function to check if an error is a timeout error -func isTimeoutError(err error) bool { - if err == nil { - return false - } - return err.Error() == "context deadline exceeded" -} \ No newline at end of file From cbcca6583f0a28c6d9f592957bfb1163968700fc Mon Sep 17 00:00:00 2001 From: cah-patrickthiem Date: Fri, 15 Nov 2024 11:47:28 +0100 Subject: [PATCH 5/6] added/adjusted the remaining tests --- .gitignore | 3 - .../event-ratelimit-config.yaml | 8 - .../kaas/kaas-sonobuoy-tests/kind_config.yaml | 2 + .../scs_0215_v1_robustness_features_test.go | 1051 +++++------------ 4 files changed, 274 insertions(+), 790 deletions(-) delete mode 100644 Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml diff --git a/.gitignore b/.gitignore index 4abd980fc..2b83a0983 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,6 @@ .sandbox .DS_Store node_modules -<<<<<<< HEAD Tests/kaas/results/ -======= ->>>>>>> 4e8fc4d (add results path to .gitignore) Tests/kaas/kaas-sonobuoy-tests/results/ *.tar.gz diff --git a/Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml b/Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml deleted file mode 100644 index 7dc0513a7..000000000 --- a/Tests/kaas/kaas-sonobuoy-tests/event-ratelimit-config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# event-ratelimit-config.yaml -kind: Configuration -apiVersion: eventratelimit.admission.k8s.io/v1alpha1 -limits: -- burst: 20000 - qps: 5000 - type: Server - diff --git a/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml b/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml index 371c74d4c..5a612f800 100644 --- a/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml +++ b/Tests/kaas/kaas-sonobuoy-tests/kind_config.yaml @@ -5,4 +5,6 @@ networking: apiServerPort: 6443 nodes: - role: control-plane +- role: worker +- role: worker - role: worker \ No newline at end of file diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go index e5d8022c4..9c81874fb 100644 --- a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go +++ b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go @@ -1,826 +1,319 @@ package scs_k8s_tests import ( - "context" - "crypto/x509" - "encoding/pem" - "fmt" - "reflect" - "log" - "os" - "strconv" - "strings" - "sync" - "testing" - "time" - - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" -) - -// Configuration constants -const ( - defaultQPS = 100 - defaultBurst = 200 + "context" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) // ==================== Helper Functions ==================== -// Cluster Information Helpers - -// getClusterSize returns the total number of nodes in the cluster -func getClusterSize(clientset *kubernetes.Clientset) int { - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - log.Fatalf("Failed to list nodes: %v", err) - } - return len(nodes.Items) -} - -// isKindCluster determines if we're running on a kind cluster by checking node labels +// isKindCluster determines if we're running on a kind cluster func isKindCluster(clientset *kubernetes.Clientset) bool { - nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) - if err != nil { - return false - } - for _, node := range nodes.Items { - if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok { - return true - } - } - return false -} - -// Configuration and Environment Helpers - -// getEnvOrDefault retrieves an environment variable or returns a default value -func getEnvOrDefault(key string, defaultValue int) int { - if value, exists := os.LookupEnv(key); exists { - if intValue, err := strconv.Atoi(value); err == nil { - return intValue - } - } - return defaultValue -} - -// setupClientset creates a Kubernetes clientset with specified QPS and Burst -func setupClientset(qps, burst float32) (*kubernetes.Clientset, error) { - config, err := rest.InClusterConfig() - if err != nil { - return nil, fmt.Errorf("failed to load in-cluster config: %v", err) - } - - config.QPS = qps - config.Burst = int(burst) - - clientset, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, fmt.Errorf("failed to create Kubernetes clientset: %v", err) - } - - return clientset, nil -} - -// Error Detection Helpers - -// isRateLimitError checks if an error is related to rate limiting -func isRateLimitError(err error) bool { - if err == nil { - return false - } - return strings.Contains(err.Error(), "429") || - strings.Contains(strings.ToLower(err.Error()), "too many requests") || - strings.Contains(strings.ToLower(err.Error()), "rate limit") + return false } -// isTimeoutError checks if an error is related to timeouts -func isTimeoutError(err error) bool { - if err == nil { - return false - } - return err == context.DeadlineExceeded || - strings.Contains(err.Error(), "context deadline exceeded") || - strings.Contains(strings.ToLower(err.Error()), "timeout") -} - -// Certificate Handling Helper - -// parseCertificate decodes and parses an X.509 certificate from PEM format -func parseCertificate(certData []byte) (*x509.Certificate, error) { - block, _ := pem.Decode(certData) - if block == nil { - return nil, fmt.Errorf("failed to decode PEM block") - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse certificate: %v", err) - } - - return cert, nil -} - -// runConcurrentRequests executes multiple concurrent API requests and counts errors -func runConcurrentRequests(clientset *kubernetes.Clientset, requestCount int) int { - var wg sync.WaitGroup - errChan := make(chan error, requestCount) - - for i := 0; i < requestCount; i++ { - wg.Add(1) - go func() { - defer wg.Done() - ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) - defer cancel() - _, err := clientset.CoreV1().Pods("").List(ctx, metav1.ListOptions{}) - if err != nil { - errChan <- err - } - }() - } - - wg.Wait() - close(errChan) - return len(errChan) -} - -// generateComplexFieldSelector creates a complex field selector for testing timeouts -func generateComplexFieldSelector() []string { - selectors := make([]string, 100) - for i := 0; i < 100; i++ { - selectors[i] = fmt.Sprintf("metadata.name!=%d", i) - } - return selectors -} - -// calculateMaxMutatingRequestsInflight determines the maximum number of mutating requests -func calculateMaxMutatingRequestsInflight(clusterSize int) int { - return getEnvOrDefault("MAX_MUTATING_REQUESTS_INFLIGHT", clusterSize*50) -} - -// hasEtcdBackup is a generic helper to check for etcd backup in various resources -func hasEtcdBackup[T any](items []T, nameCheck func(string) bool) bool { - for _, item := range items { - name := reflect.ValueOf(item).FieldByName("Name").String() - if nameCheck(name) { - return true - } - } - return false -} - -// findCertificateData looks for certificate data in secret data map -func findCertificateData(data map[string][]byte) []byte { - // First check for standard TLS certificate key - if certData, ok := data["tls.crt"]; ok { - return certData - } - - // Try alternative key names - for key, data := range data { - if strings.Contains(key, ".crt") || strings.Contains(key, "cert") { - return data - } - } - return nil +// setupClientset creates a Kubernetes clientset +func setupClientset() (*kubernetes.Clientset, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to load in-cluster config: %v", err) + } + return kubernetes.NewForConfig(config) } // ==================== Test Cases ==================== -// Test groups are organized by feature: -// 1. API Request Limits -// 2. Request Timeout -// 3. Event Rate Limiting -// 4. API Priority and Fairness -// 5. etcd Management -// 6. Certificate Management - -// -------------------- 1. API Request Limits -------------------- - -// Test_scs_0215_maxRequestInflight verifies the API server's max-requests-inflight setting -func Test_scs_0215_maxRequestInflight(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - clusterSize := getClusterSize(clientset) - maxRequestsInflight := getEnvOrDefault("MAX_REQUESTS_INFLIGHT", clusterSize*250) - - t.Logf("Detected cluster size: %d nodes", clusterSize) - t.Logf("Using maxRequestsInflight = %d", maxRequestsInflight) - - t.Run("Positive_Test_Case", func(t *testing.T) { - // Test with safe number of requests (25% of limit) - t.Log("Running Positive Test Case") - safeRequests := maxRequestsInflight / 4 - errors := runConcurrentRequests(clientset, safeRequests) - if errors > 0 { - t.Errorf("Test failed: encountered %d unexpected errors when requests were expected to succeed.", errors) - } else { - t.Log("Positive test case passed successfully!") - } - }) - - t.Run("Negative_Test_Case", func(t *testing.T) { - // Test with excessive requests (200% of limit) - t.Log("Running Negative Test Case") - overloadRequests := maxRequestsInflight * 2 - errors := runConcurrentRequests(clientset, overloadRequests) - - if errors == 0 { - t.Error("Test failed: expected rate limit errors, but all requests succeeded") - } else { - t.Log("Negative test case passed as expected: rate limit exceeded") - } - }) +func Test_scs_0215_requestLimits(t *testing.T) { + clientset, err := setupClientset() + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + t.Run("Check_Request_Limit_Configuration", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + pods, err := clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil || len(pods.Items) == 0 { + t.Fatalf("Failed to find kube-apiserver pod: %v", err) + } + + apiServer := pods.Items[0] + var foundSettings = make(map[string]bool) + requiredSettings := []string{ + "max-requests-inflight", + "max-mutating-requests-inflight", + "min-request-timeout", + } + + for _, container := range apiServer.Spec.Containers { + for _, arg := range container.Command { + for _, setting := range requiredSettings { + if strings.Contains(arg, setting) { + foundSettings[setting] = true + } + } + if strings.Contains(arg, "enable-admission-plugins") && + strings.Contains(arg, "EventRateLimit") { + foundSettings["EventRateLimit"] = true + } + } + } + + for _, setting := range requiredSettings { + if !foundSettings[setting] { + t.Errorf("Required setting %s not found in API server configuration", setting) + } + } + if !foundSettings["EventRateLimit"] { + t.Error("EventRateLimit admission plugin not enabled") + } + }) } -// Test_scs_0215_maxMutatingRequestsInflight verifies the API server's max-mutating-requests-inflight setting -func Test_scs_0215_maxMutatingRequestsInflight(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - clusterSize := getClusterSize(clientset) - maxMutatingRequestsInflight := calculateMaxMutatingRequestsInflight(clusterSize) - isKind := isKindCluster(clientset) - - t.Logf("Detected cluster size: %d nodes", clusterSize) - t.Logf("Using maxMutatingRequestsInflight = %d", maxMutatingRequestsInflight) - - t.Run("Positive_Test_Case", func(t *testing.T) { - t.Log("Running Positive Test Case") - err := runMutatingTest(clientset, maxMutatingRequestsInflight/2) - if err != nil { - t.Fatalf("Positive test failed: %v", err) - } - t.Log("Positive test case passed successfully!") - }) - - t.Run("Negative_Test_Case", func(t *testing.T) { - t.Log("Running Negative Test Case") - // Use higher QPS/burst for negative test case - highLoadClientset, _ := setupClientset(2000, 2000) - - var wg sync.WaitGroup - errorChan := make(chan error, 5) - - // Run multiple concurrent batches of requests - batchCount := 5 - if isKind { - batchCount = 2 // Use fewer batches for kind clusters - } - - for i := 0; i < batchCount; i++ { - wg.Add(1) - go func() { - defer wg.Done() - err := runMutatingTest(highLoadClientset, maxMutatingRequestsInflight) - if err != nil { - errorChan <- err - } - }() - } - - wg.Wait() - close(errorChan) - - var gotRateLimit bool - for err := range errorChan { - if isRateLimitError(err) { - gotRateLimit = true - break - } - } - - if !gotRateLimit { - if isKind { - t.Log("No rate limit error in kind cluster - this is acceptable") - } else { - t.Error("Expected rate limit errors in production cluster, but got none") - } - } - }) -} - -// runMutatingTest performs concurrent mutating requests to test rate limiting -func runMutatingTest(clientset *kubernetes.Clientset, limit int) error { - var wg sync.WaitGroup - errChan := make(chan error, limit) - - for i := 0; i < limit; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - ctx, cancel := context.WithTimeout(context.TODO(), 2*time.Second) - defer cancel() - - _, err := clientset.CoreV1().ConfigMaps("default").Create(ctx, &v1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: fmt.Sprintf("test-cm-%d-", i), - }, - Data: map[string]string{ - "test": fmt.Sprintf("data-%d", i), - "load": strings.Repeat("x", 1000), // Add some data to increase request size - }, - }, metav1.CreateOptions{}) - - if err != nil { - errChan <- err - } - }(i) - } - - wg.Wait() - close(errChan) - - var lastError error - for err := range errChan { - if isRateLimitError(err) { - return err - } - lastError = err - } - - return lastError -} - -// -------------------- 2. Request Timeout -------------------- - -// Test_scs_0215_minRequestTimeout verifies the API server's min-request-timeout setting func Test_scs_0215_minRequestTimeout(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) + clientset, err := setupClientset() if err != nil { t.Fatalf("Failed to setup clientset: %v", err) } - t.Run("Test_minRequestTimeout", func(t *testing.T) { - isKind := isKindCluster(clientset) - minRequestTimeout := time.Duration(getEnvOrDefault("MIN_REQUEST_TIMEOUT", 1)) * time.Second - t.Logf("Testing with min-request-timeout = %v", minRequestTimeout) - - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - startTime := time.Now() - _, err := clientset.CoreV1().Pods("default").List(ctx, metav1.ListOptions{ - FieldSelector: strings.Join(generateComplexFieldSelector(), ","), + t.Run("Check_minRequestTimeout_Configuration", func(t *testing.T) { + pods, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{ + LabelSelector: "component=kube-apiserver", }) - duration := time.Since(startTime) - - if isKind { - t.Logf("Running on kind cluster. Request completed in %v", duration) - } else { - if err == nil && duration < minRequestTimeout { - t.Errorf("Request completed faster than minimum timeout. Duration: %v, Expected minimum: %v", duration, minRequestTimeout) - } else { - t.Logf("Request completed as expected in %v", duration) - } - } - }) -} - -// -------------------- 3. Event Rate Limiting -------------------- - -// Test_scs_0215_eventRateLimit verifies the EventRateLimit admission controller configuration -func Test_scs_0215_eventRateLimit(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - isKind := isKindCluster(clientset) - if isKind { - t.Skip("Running on kind cluster - skipping EventRateLimit test as it's not required for development environments") - } - - t.Run("Check_EventRateLimit_Configuration", func(t *testing.T) { - // Check configurations in multiple locations - configLocations := []struct { - name string - namespace string - key string - }{ - {"admission-configuration", "kube-system", "eventratelimit.yaml"}, - {"kube-apiserver", "kube-system", "config.yaml"}, + if err != nil || len(pods.Items) == 0 { + t.Fatalf("Failed to find kube-apiserver pod: %v", err) } - for _, loc := range configLocations { - config, err := clientset.CoreV1().ConfigMaps(loc.namespace).Get( - context.TODO(), - loc.name, - metav1.GetOptions{}, - ) - if err == nil { - if data, ok := config.Data[loc.key]; ok { - if strings.Contains(data, "eventratelimit.admission.k8s.io") { - t.Logf("Found EventRateLimit configuration in %s/%s", loc.namespace, loc.name) - return - } + found := false + for _, container := range pods.Items[0].Spec.Containers { + for _, arg := range container.Command { + if strings.Contains(arg, "--min-request-timeout=") { + found = true + break } } } - // Check for standalone configuration - configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.TODO(), metav1.ListOptions{}) - for _, cm := range configMaps.Items { - if strings.Contains(cm.Name, "event-rate-limit") { - t.Logf("Found standalone EventRateLimit configuration in ConfigMap: %s", cm.Name) - return - } + if !found { + t.Error("min-request-timeout not configured for API server") } - - t.Error("No EventRateLimit configuration found in production cluster") }) } -// -------------------- 4. API Priority and Fairness -------------------- +func Test_scs_0215_eventRateLimit(t *testing.T) { + clientset, err := setupClientset() + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + if isKindCluster(clientset) { + t.Skip("Running on kind cluster - skipping EventRateLimit test") + } + + t.Run("Check_EventRateLimit_Configuration", func(t *testing.T) { + configLocations := []struct { + name string + namespace string + key string + }{ + {"admission-configuration", "kube-system", "eventratelimit.yaml"}, + {"kube-apiserver", "kube-system", "config.yaml"}, + } + + for _, loc := range configLocations { + config, err := clientset.CoreV1().ConfigMaps(loc.namespace).Get(context.Background(), loc.name, metav1.GetOptions{}) + if err == nil { + if data, ok := config.Data[loc.key]; ok { + if strings.Contains(data, "eventratelimit.admission.k8s.io") { + t.Logf("Found EventRateLimit configuration in %s/%s", loc.namespace, loc.name) + return + } + } + } + } + + configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.Background(), metav1.ListOptions{}) + for _, cm := range configMaps.Items { + if strings.Contains(cm.Name, "event-rate-limit") { + t.Logf("Found standalone EventRateLimit configuration in ConfigMap: %s", cm.Name) + return + } + } + + t.Error("No EventRateLimit configuration found in production cluster") + }) +} -// Test_scs_0215_apiPriorityAndFairness verifies API Priority and Fairness (APF) configuration func Test_scs_0215_apiPriorityAndFairness(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - isKind := isKindCluster(clientset) - if isKind { - t.Skip("Running on kind cluster - skipping APF test as it's configured differently in development environments") - } - - t.Run("Check_APF_Configuration", func(t *testing.T) { - // Multiple checks for APF configuration - checks := []struct { - name string - fn func() bool - }{ - {"API Server Config", func() bool { - config, err := clientset.CoreV1().ConfigMaps("kube-system").Get( - context.TODO(), - "kube-apiserver", - metav1.GetOptions{}, - ) - return err == nil && config.Data["config.yaml"] != "" && - strings.Contains(config.Data["config.yaml"], "enable-priority-and-fairness: true") - }}, - {"Command Line Flags", func() bool { - pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ - LabelSelector: "component=kube-apiserver", - }) - if err != nil || len(pods.Items) == 0 { - return false - } - for _, pod := range pods.Items { - for _, container := range pod.Spec.Containers { - for _, arg := range container.Command { - if strings.Contains(arg, "--enable-priority-and-fairness=true") { - return true - } - } - } - } - return false - }}, - {"API Resources", func() bool { - resources, err := clientset.Discovery().ServerResourcesForGroupVersion("flowcontrol.apiserver.k8s.io/v1beta3") - if err != nil || resources == nil { - return false - } - for _, r := range resources.APIResources { - if r.Name == "flowschemas" || r.Name == "prioritylevelconfigurations" { - return true - } - } - return false - }}, - } - - for _, check := range checks { - if check.fn() { - t.Logf("APF enabled via %s", check.name) - return - } - } - - t.Error("No API Priority and Fairness configuration found in production cluster") - }) + clientset, err := setupClientset() + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + if isKindCluster(clientset) { + t.Skip("Running on kind cluster - skipping APF test") + } + + t.Run("Check_APF_Configuration", func(t *testing.T) { + pods, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{ + LabelSelector: "component=kube-apiserver", + }) + if err != nil || len(pods.Items) == 0 { + t.Fatal("Failed to find kube-apiserver pod") + } + + for _, container := range pods.Items[0].Spec.Containers { + for _, arg := range container.Command { + if strings.Contains(arg, "--enable-priority-and-fairness=true") { + t.Log("APF enabled via Command Line Flags") + return + } + } + } + + t.Error("API Priority and Fairness not enabled") + }) } -// Test_scs_0215_rateLimitValues verifies recommended rate limit values func Test_scs_0215_rateLimitValues(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - isKind := isKindCluster(clientset) - if isKind { - t.Skip("Running on kind cluster - skipping rate limit values test") - } - - t.Run("Check_Rate_Limit_Values", func(t *testing.T) { - // Define expected values - expectedValues := map[string]string{ - "qps": "5000", - "burst": "20000", - } - - // Check various possible configuration locations - configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.TODO(), metav1.ListOptions{}) - for _, cm := range configMaps.Items { - var config string - switch { - case strings.Contains(cm.Name, "event-rate-limit"): - config = cm.Data["config.yaml"] - case cm.Name == "admission-configuration": - config = cm.Data["eventratelimit.yaml"] - case cm.Name == "kube-apiserver": - config = cm.Data["config.yaml"] - } - - if config != "" { - allFound := true - for k, v := range expectedValues { - if !strings.Contains(config, fmt.Sprintf("%s: %s", k, v)) { - allFound = false - break - } - } - if allFound { - t.Log("Found recommended rate limit values") - return - } - } - } - - t.Error("Recommended rate limit values (qps: 5000, burst: 20000) not found") - }) + clientset, err := setupClientset() + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + if isKindCluster(clientset) { + t.Skip("Running on kind cluster - skipping rate limit values test") + } + + t.Run("Check_Rate_Limit_Values", func(t *testing.T) { + expectedValues := map[string]string{ + "qps": "5000", + "burst": "20000", + } + + configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.Background(), metav1.ListOptions{}) + for _, cm := range configMaps.Items { + var config string + switch { + case strings.Contains(cm.Name, "event-rate-limit"): + config = cm.Data["config.yaml"] + case cm.Name == "admission-configuration": + config = cm.Data["eventratelimit.yaml"] + case cm.Name == "kube-apiserver": + config = cm.Data["config.yaml"] + } + + if config != "" { + allFound := true + for k, v := range expectedValues { + if !strings.Contains(config, fmt.Sprintf("%s: %s", k, v)) { + allFound = false + break + } + } + if allFound { + return + } + } + } + + t.Error("Recommended rate limit values (qps: 5000, burst: 20000) not found") + }) } -// -------------------- 5. etcd Management -------------------- - -// Test_scs_0215_etcdCompaction verifies etcd compaction settings func Test_scs_0215_etcdCompaction(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - isKind := isKindCluster(clientset) - if isKind { - t.Skip("Running on kind cluster - skipping etcd compaction test as it has different default settings") - } - - t.Run("Check_Etcd_Compaction_Settings", func(t *testing.T) { - // Try different label selectors for etcd pods - selectors := []string{ - "component=etcd", - "k8s-app=etcd", - "tier=control-plane,component=etcd", - } - - var etcdPods []v1.Pod - for _, selector := range selectors { - pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ - LabelSelector: selector, - }) - if err == nil && len(pods.Items) > 0 { - etcdPods = pods.Items - t.Logf("Found etcd pods using selector: %s", selector) - break - } - } - - if len(etcdPods) == 0 { - t.Skip("No etcd pods found - they might be running outside the cluster") - return - } - - // Expected settings - requiredSettings := map[string]string{ - "auto-compaction-mode": "periodic", - "auto-compaction-retention": "8h", - } - - // Check each etcd pod - for _, pod := range etcdPods { - t.Logf("Checking etcd pod: %s", pod.Name) - - // Check ConfigMap first - if cm, err := clientset.CoreV1().ConfigMaps("kube-system").Get( - context.TODO(), - "etcd-config", - metav1.GetOptions{}, - ); err == nil { - if config, ok := cm.Data["etcd.conf.yaml"]; ok { - allFound := true - for setting, value := range requiredSettings { - if !strings.Contains(config, fmt.Sprintf("%s: %s", setting, value)) { - allFound = false - break - } - } - if allFound { - t.Log("Found correct etcd compaction settings in ConfigMap") - return - } - } - } - - // Check command line arguments - for _, container := range pod.Spec.Containers { - foundSettings := make(map[string]bool) - for _, arg := range container.Command { - for setting, value := range requiredSettings { - if strings.Contains(arg, fmt.Sprintf("--%s=%s", setting, value)) { - foundSettings[setting] = true - } - } - } - - if len(foundSettings) == len(requiredSettings) { - t.Log("Found correct etcd compaction settings in command line arguments") - return - } - } - } - - t.Error("Required etcd compaction settings not found") - }) + clientset, err := setupClientset() + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + if isKindCluster(clientset) { + t.Skip("Running on kind cluster - skipping etcd compaction test") + } + + t.Run("Check_Etcd_Compaction_Settings", func(t *testing.T) { + pods, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{ + LabelSelector: "component=etcd", + }) + if err != nil || len(pods.Items) == 0 { + t.Skip("No etcd pods found") + return + } + + requiredSettings := map[string]string{ + "auto-compaction-mode": "periodic", + "auto-compaction-retention": "8h", + } + + for _, pod := range pods.Items { + for _, container := range pod.Spec.Containers { + foundSettings := make(map[string]bool) + for _, arg := range container.Command { + for setting, value := range requiredSettings { + if strings.Contains(arg, fmt.Sprintf("--%s=%s", setting, value)) { + foundSettings[setting] = true + } + } + } + + if len(foundSettings) == len(requiredSettings) { + t.Log("Found correct etcd compaction settings") + return + } + } + } + + t.Error("Required etcd compaction settings not found") + }) } -// Test_scs_0215_etcdBackup verifies etcd backup configuration func Test_scs_0215_etcdBackup(t *testing.T) { - clientset, err := setupClientset(defaultQPS, defaultBurst) - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - isKind := isKindCluster(clientset) - if isKind { - t.Skip("Running on kind cluster - skipping etcd backup test as it's typically handled differently in development environments") - } - - t.Run("Check_Etcd_Backup_Configuration", func(t *testing.T) { - // Define backup solution checks - backupChecks := []struct { - name string - fn func() bool - }{ - {"CronJob", func() bool { - cronJobs, err := clientset.BatchV1().CronJobs("").List(context.TODO(), metav1.ListOptions{}) - return err == nil && hasEtcdBackup(cronJobs.Items, func(name string) bool { - return strings.Contains(strings.ToLower(name), "etcd") && - strings.Contains(strings.ToLower(name), "backup") - }) - }}, - {"Deployment", func() bool { - deployments, err := clientset.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{}) - return err == nil && hasEtcdBackup(deployments.Items, func(name string) bool { - return strings.Contains(strings.ToLower(name), "etcd") && - strings.Contains(strings.ToLower(name), "backup") - }) - }}, - {"DaemonSet", func() bool { - daemonSets, err := clientset.AppsV1().DaemonSets("").List(context.TODO(), metav1.ListOptions{}) - return err == nil && hasEtcdBackup(daemonSets.Items, func(name string) bool { - return strings.Contains(strings.ToLower(name), "etcd") && - strings.Contains(strings.ToLower(name), "backup") - }) - }}, - {"Dedicated Pods", func() bool { - pods, err := clientset.CoreV1().Pods("").List(context.TODO(), metav1.ListOptions{}) - return err == nil && hasEtcdBackup(pods.Items, func(name string) bool { - return strings.Contains(strings.ToLower(name), "etcd") && - strings.Contains(strings.ToLower(name), "backup") - }) - }}, - } - - for _, check := range backupChecks { - if check.fn() { - t.Logf("Found etcd backup solution: %s", check.name) - return - } - } - - t.Error("No etcd backup solution found. Required: at least weekly backups through CronJob, Deployment, DaemonSet, or dedicated Pods") - }) + clientset, err := setupClientset() + if err != nil { + t.Fatalf("Failed to setup clientset: %v", err) + } + + if isKindCluster(clientset) { + t.Skip("Running on kind cluster - skipping etcd backup test") + } + + t.Run("Check_Etcd_Backup_Configuration", func(t *testing.T) { + cronJobs, err := clientset.BatchV1().CronJobs("").List(context.Background(), metav1.ListOptions{}) + if err == nil { + for _, job := range cronJobs.Items { + if strings.Contains(strings.ToLower(job.Name), "etcd") && + strings.Contains(strings.ToLower(job.Name), "backup") { + t.Log("Found etcd backup solution: CronJob") + return + } + } + } + t.Error("No etcd backup solution found") + }) } -// -------------------- 6. Certificate Management -------------------- - -// Test_scs_0215_certificateRotation verifies certificate rotation configuration func Test_scs_0215_certificateRotation(t *testing.T) { - clientset, err := setupClientset(100, 200) // Using default QPS and Burst - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - isKind := isKindCluster(clientset) - if isKind { - t.Skip("Running on kind cluster - skipping certificate rotation test") - } - - t.Run("Check_Certificate_Expiration", func(t *testing.T) { - secrets, err := clientset.CoreV1().Secrets("kube-system").List(context.TODO(), metav1.ListOptions{}) - if err != nil { - t.Fatalf("Failed to list secrets: %v", err) - } - - oneYearFromNow := time.Now().AddDate(1, 0, 0) - certsChecked := 0 - - for _, secret := range secrets.Items { - // Check for TLS secrets and certificate-related secrets - if secret.Type == v1.SecretTypeTLS || - strings.Contains(secret.Name, "cert") || - strings.Contains(secret.Name, "certificate") { - - certData := findCertificateData(secret.Data) - if certData == nil { - continue - } - - cert, err := parseCertificate(certData) - if err != nil { - t.Logf("Failed to parse certificate from secret %s: %v", secret.Name, err) - continue - } - - certsChecked++ - t.Logf("Checking certificate in secret: %s, expires: %v", secret.Name, cert.NotAfter) - - if cert.NotAfter.Before(oneYearFromNow) { - t.Errorf("Certificate in secret %s expires in less than a year (expires: %v)", - secret.Name, cert.NotAfter) - } - } - } - - if certsChecked == 0 { - t.Error("No certificates found to check") - } else { - t.Logf("Checked %d certificates", certsChecked) - } - }) - - t.Run("Check_Certificate_Controller", func(t *testing.T) { - // Check for cert-manager deployment - deployments, err := clientset.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{}) - if err != nil { - t.Fatalf("Failed to list deployments: %v", err) - } - - // Look for various certificate controller implementations - certControllers := []string{ - "cert-manager", - "certificate-controller", - "cert-controller", - } - - found := false - for _, deployment := range deployments.Items { - for _, controller := range certControllers { - if strings.Contains(strings.ToLower(deployment.Name), controller) { - if deployment.Status.ReadyReplicas > 0 { - found = true - t.Logf("Found active certificate controller: %s", deployment.Name) - break - } - } - } - if found { - break - } - } - - if !found { - // Check for built-in controller in kube-controller-manager - pods, err := clientset.CoreV1().Pods("kube-system").List(context.TODO(), metav1.ListOptions{ - LabelSelector: "component=kube-controller-manager", - }) - if err == nil && len(pods.Items) > 0 { - for _, pod := range pods.Items { - for _, container := range pod.Spec.Containers { - for _, arg := range container.Command { - if strings.Contains(arg, "--controllers=*") || - strings.Contains(arg, "--controllers=certificate") { - found = true - t.Log("Found built-in certificate controller in kube-controller-manager") - break - } - } - } - } - } - } - - if !found { - t.Error("No certificate controller found") - } - }) + clientset, err := setupClientset() + if err != nil { + t.Skip("Failed to setup clientset") + } + + t.Run("Check_Certificate_Controller", func(t *testing.T) { + _, err := clientset.AppsV1().Deployments("cert-manager").Get(context.Background(), "cert-manager", metav1.GetOptions{}) + if err != nil { + t.Error("cert-manager not found - certificate controller required") + } else { + t.Log("Found active certificate controller: cert-manager") + } + }) } \ No newline at end of file From cb8e7a3e8af785831fd75f8951a90c668eea0e5c Mon Sep 17 00:00:00 2001 From: cah-patrickthiem Date: Thu, 5 Dec 2024 15:14:39 +0100 Subject: [PATCH 6/6] Addressed review commands. --- .../scs_0215_v1_robustness_features_test.go | 113 +----------------- 1 file changed, 1 insertion(+), 112 deletions(-) diff --git a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go index 9c81874fb..a473acd68 100644 --- a/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go +++ b/Tests/kaas/kaas-sonobuoy-tests/scs_k8s_conformance_tests/scs_0215_v1_robustness_features_test.go @@ -14,11 +14,6 @@ import ( // ==================== Helper Functions ==================== -// isKindCluster determines if we're running on a kind cluster -func isKindCluster(clientset *kubernetes.Clientset) bool { - return false -} - // setupClientset creates a Kubernetes clientset func setupClientset() (*kubernetes.Clientset, error) { config, err := rest.InClusterConfig() @@ -53,6 +48,7 @@ func Test_scs_0215_requestLimits(t *testing.T) { "max-requests-inflight", "max-mutating-requests-inflight", "min-request-timeout", + "EventRateLimit", } for _, container := range apiServer.Spec.Containers { @@ -74,9 +70,6 @@ func Test_scs_0215_requestLimits(t *testing.T) { t.Errorf("Required setting %s not found in API server configuration", setting) } } - if !foundSettings["EventRateLimit"] { - t.Error("EventRateLimit admission plugin not enabled") - } }) } @@ -116,10 +109,6 @@ func Test_scs_0215_eventRateLimit(t *testing.T) { t.Fatalf("Failed to setup clientset: %v", err) } - if isKindCluster(clientset) { - t.Skip("Running on kind cluster - skipping EventRateLimit test") - } - t.Run("Check_EventRateLimit_Configuration", func(t *testing.T) { configLocations := []struct { name string @@ -160,10 +149,6 @@ func Test_scs_0215_apiPriorityAndFairness(t *testing.T) { t.Fatalf("Failed to setup clientset: %v", err) } - if isKindCluster(clientset) { - t.Skip("Running on kind cluster - skipping APF test") - } - t.Run("Check_APF_Configuration", func(t *testing.T) { pods, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{ LabelSelector: "component=kube-apiserver", @@ -185,108 +170,12 @@ func Test_scs_0215_apiPriorityAndFairness(t *testing.T) { }) } -func Test_scs_0215_rateLimitValues(t *testing.T) { - clientset, err := setupClientset() - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - if isKindCluster(clientset) { - t.Skip("Running on kind cluster - skipping rate limit values test") - } - - t.Run("Check_Rate_Limit_Values", func(t *testing.T) { - expectedValues := map[string]string{ - "qps": "5000", - "burst": "20000", - } - - configMaps, _ := clientset.CoreV1().ConfigMaps("kube-system").List(context.Background(), metav1.ListOptions{}) - for _, cm := range configMaps.Items { - var config string - switch { - case strings.Contains(cm.Name, "event-rate-limit"): - config = cm.Data["config.yaml"] - case cm.Name == "admission-configuration": - config = cm.Data["eventratelimit.yaml"] - case cm.Name == "kube-apiserver": - config = cm.Data["config.yaml"] - } - - if config != "" { - allFound := true - for k, v := range expectedValues { - if !strings.Contains(config, fmt.Sprintf("%s: %s", k, v)) { - allFound = false - break - } - } - if allFound { - return - } - } - } - - t.Error("Recommended rate limit values (qps: 5000, burst: 20000) not found") - }) -} - -func Test_scs_0215_etcdCompaction(t *testing.T) { - clientset, err := setupClientset() - if err != nil { - t.Fatalf("Failed to setup clientset: %v", err) - } - - if isKindCluster(clientset) { - t.Skip("Running on kind cluster - skipping etcd compaction test") - } - - t.Run("Check_Etcd_Compaction_Settings", func(t *testing.T) { - pods, err := clientset.CoreV1().Pods("kube-system").List(context.Background(), metav1.ListOptions{ - LabelSelector: "component=etcd", - }) - if err != nil || len(pods.Items) == 0 { - t.Skip("No etcd pods found") - return - } - - requiredSettings := map[string]string{ - "auto-compaction-mode": "periodic", - "auto-compaction-retention": "8h", - } - - for _, pod := range pods.Items { - for _, container := range pod.Spec.Containers { - foundSettings := make(map[string]bool) - for _, arg := range container.Command { - for setting, value := range requiredSettings { - if strings.Contains(arg, fmt.Sprintf("--%s=%s", setting, value)) { - foundSettings[setting] = true - } - } - } - - if len(foundSettings) == len(requiredSettings) { - t.Log("Found correct etcd compaction settings") - return - } - } - } - - t.Error("Required etcd compaction settings not found") - }) -} - func Test_scs_0215_etcdBackup(t *testing.T) { clientset, err := setupClientset() if err != nil { t.Fatalf("Failed to setup clientset: %v", err) } - if isKindCluster(clientset) { - t.Skip("Running on kind cluster - skipping etcd backup test") - } - t.Run("Check_Etcd_Backup_Configuration", func(t *testing.T) { cronJobs, err := clientset.BatchV1().CronJobs("").List(context.Background(), metav1.ListOptions{}) if err == nil {