From 75ba5d3b0bbea57605c1b9eb36df26db21b2917f Mon Sep 17 00:00:00 2001 From: ComradeProgrammer Date: Sun, 9 Nov 2025 19:32:56 +0100 Subject: [PATCH] [CI] add docker-docker-linux-x86-64 integration test in prow --- .../integration_docker_docker_linux_x86-64.sh | 37 +++++++++++++++++++ .../integration_kvm_docker_linux_x86-64.sh | 2 +- hack/prow/prow.mk | 7 ++++ hack/prow/util/integration_prow_wrapper.sh | 30 +++++++++++++++ hack/prow/util/run_with_minikube_user.sh | 37 +++++++++++++++++++ test/integration/scheduled_stop_test.go | 21 +++++++++-- 6 files changed, 129 insertions(+), 5 deletions(-) create mode 100755 hack/prow/integration_docker_docker_linux_x86-64.sh create mode 100755 hack/prow/util/integration_prow_wrapper.sh create mode 100755 hack/prow/util/run_with_minikube_user.sh diff --git a/hack/prow/integration_docker_docker_linux_x86-64.sh b/hack/prow/integration_docker_docker_linux_x86-64.sh new file mode 100755 index 000000000000..b82bba73874f --- /dev/null +++ b/hack/prow/integration_docker_docker_linux_x86-64.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2025 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +OS="linux" +ARCH="amd64" +DRIVER="docker" +CONTAINER_RUNTIME="docker" +EXTRA_START_ARGS="" +EXTRA_TEST_ARGS="" +JOB_NAME="Docker_Linux" + +git config --global --add safe.directory '*' +COMMIT=$(git rev-parse HEAD) +MINIKUBE_LOCATION=$COMMIT + + +# when docker is the driver, we run integration tests directly in prow cluster +# by default, prow jobs run in root, so we must switch to a non-root user to run docker driver + + +source ./hack/prow/common.sh diff --git a/hack/prow/integration_kvm_docker_linux_x86-64.sh b/hack/prow/integration_kvm_docker_linux_x86-64.sh index af4782fc8c53..031904345b65 100755 --- a/hack/prow/integration_kvm_docker_linux_x86-64.sh +++ b/hack/prow/integration_kvm_docker_linux_x86-64.sh @@ -25,7 +25,7 @@ CONTAINER_RUNTIME="docker" EXTRA_START_ARGS="" EXTRA_TEST_ARGS="" JOB_NAME="KVM_Linux" - +git config --global --add safe.directory '*' COMMIT=$(git rev-parse HEAD) MINIKUBE_LOCATION=$COMMIT echo "running test in $(pwd)" diff --git a/hack/prow/prow.mk b/hack/prow/prow.mk index 20f7798522f8..47004ad925c8 100644 --- a/hack/prow/prow.mk +++ b/hack/prow/prow.mk @@ -1,5 +1,12 @@ .PHONY: integration-prow-kvm-docker-linux-x86-64 + +integration-prow-docker-docker-linux-x86-64: +# build first +# container-runtime=docker driver=docker on linux/amd64 + ./hack/prow/minikube_cross_build.sh $(GO_VERSION) linux amd64 + ./hack/prow/util/integration_prow_wrapper.sh ./hack/prow/integration_docker_docker_linux_x86-64.sh + integration-prow-kvm-docker-linux-x86-64: # build first # container-runtime=docker driver=kvm on linux/amd64 diff --git a/hack/prow/util/integration_prow_wrapper.sh b/hack/prow/util/integration_prow_wrapper.sh new file mode 100755 index 000000000000..eb3d5c705155 --- /dev/null +++ b/hack/prow/util/integration_prow_wrapper.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright 2025 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +TARGET_SCRIPT=$1 +# run the target script with minikube user +./hack/prow/util/run_with_minikube_user.sh "$TARGET_SCRIPT" +result=$? +# collect the logs as root user +echo "test finished with exit code $result" + +items=("testout.txt" "test.json" "junit-unit.xml" "test.html" "test_summary.json") +for item in "${items[@]}"; do + echo "Collecting ${item} to ${ARTIFACTS}/${item}" + cp "${item}" "${ARTIFACTS}/${item}" +done +exit $result diff --git a/hack/prow/util/run_with_minikube_user.sh b/hack/prow/util/run_with_minikube_user.sh new file mode 100755 index 000000000000..4d16a0675ff7 --- /dev/null +++ b/hack/prow/util/run_with_minikube_user.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Copyright 2025 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +# when docker is the driver, we run integration tests directly in prow cluster +# by default, prow jobs run in root, so we must switch to a non-root user to run docker driver +NEW_USER="minikube" +TARGET_SCRIPT=$1 + +if [ "$(whoami)" == "root" ]; then + useradd -m -s /bin/bash "$NEW_USER" +fi +chown -R "$NEW_USER":"$NEW_USER" . +# install sudo if not present +apt-get update && apt-get install -y sudo +# give the new user passwordless sudo +echo "$NEW_USER ALL=(ALL) NOPASSWD:ALL" > "/etc/sudoers.d/$NEW_USER" +chmod 440 "/etc/sudoers.d/$NEW_USER" +# add the new user to the docker group +usermod -aG docker "$NEW_USER" +# exec the target script as the new user +su "$NEW_USER" -c "$TARGET_SCRIPT" \ No newline at end of file diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go index 2b0181be2ea4..cf1ae435776a 100644 --- a/test/integration/scheduled_stop_test.go +++ b/test/integration/scheduled_stop_test.go @@ -84,7 +84,7 @@ func TestScheduledStopUnix(t *testing.T) { startMinikube(ctx, t, profile) // schedule a stop for 5 min from now and make sure PID is created - stopMinikube(ctx, t, profile, []string{"--schedule", "5m"}) + stopMinikube(ctx, t, profile, []string{"--schedule", "5m", "-v=5", "--alsologtostderr"}) // make sure timeToStop is present in status ensureMinikubeScheduledTime(ctx, t, profile, 5*time.Minute) pid := checkPID(t, profile) @@ -93,7 +93,7 @@ func TestScheduledStopUnix(t *testing.T) { } // schedule a second stop which should cancel the first scheduled stop - stopMinikube(ctx, t, profile, []string{"--schedule", "15s"}) + stopMinikube(ctx, t, profile, []string{"--schedule", "15s", "-v=5", "--alsologtostderr"}) if processRunning(t, pid) { t.Fatalf("process %v running but should have been killed on reschedule of stop", pid) } @@ -109,7 +109,7 @@ func TestScheduledStopUnix(t *testing.T) { ensureTimeToStopNotPresent(ctx, t, profile) // schedule another stop, make sure minikube status is "Stopped" - stopMinikube(ctx, t, profile, []string{"--schedule", "15s"}) + stopMinikube(ctx, t, profile, []string{"--schedule", "15s", "-v=5", "--alsologtostderr"}) time.Sleep(15 * time.Second) if processRunning(t, pid) { t.Fatalf("process %v running but should have been killed on reschedule of stop", pid) @@ -138,6 +138,9 @@ func stopMinikube(ctx context.Context, t *testing.T, profile string, additionalA if err != nil { t.Fatalf("stopping minikube: %v\n%s", err, rr.Output()) } + fmt.Println("minikube stop output:") + fmt.Println(rr.Output()) + } func checkPID(t *testing.T, profile string) string { @@ -167,7 +170,17 @@ func processRunning(t *testing.T, pid string) bool { } err = process.Signal(syscall.Signal(0)) t.Log("signal error was: ", err) - return err == nil + if err != nil { + return false + } + // then check if this process has become a zombie process + // which is also not running + data, _ := os.ReadFile(fmt.Sprintf("/proc/%s/status", pid)) + if strings.Contains(string(data), "State:\tZ") { + t.Logf("process %s is a zombie", pid) + return false + } + return true } func ensureMinikubeStatus(ctx context.Context, t *testing.T, profile, key string, wantStatus string) { checkStatus := func() error {