Skip to content

Commit c299fd2

Browse files
ComradeProgrammerComradeProgrammer
committed
[CI] add minikubekvm integration test in prow
1 parent 5ca02f9 commit c299fd2

24 files changed

+2019
-1
lines changed

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1126,3 +1126,9 @@ get-dependency-version:
11261126
.PHONY: _update-all
11271127
_update-all:
11281128
@(cd hack && go run update/update_all/update_all.go)
1129+
1130+
1131+
1132+
# targets for tests on prow
1133+
include ./hack/prow/prow.mk
1134+

‎hack/prow/common.sh‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
#!/bin/bash
2+
3+
# Copyright 2025 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# The script expects the following env variables:
18+
# OS: The operating system
19+
# ARCH: The architecture
20+
# DRIVER: the driver to use for the test
21+
# CONTAINER_RUNTIME: the container runtime to use for the test
22+
# EXTRA_START_ARGS: additional flags to pass into minikube start
23+
# EXTRA_TEST_ARGS: additional flags to pass into go test
24+
# JOB_NAME: the name of the logfile and check name to update on github
25+
# PULL_NUMBER: the PR number, if applicable
26+
27+
function print_test_info() {
28+
echo ">> Starting at $(date)"
29+
echo ""
30+
echo "user: $(whoami)"
31+
echo "arch: ${OS_ARCH}"
32+
echo "pr: ${PULL_NUMBER}"
33+
echo "driver: ${DRIVER}"
34+
echo "runtime: ${CONTAINER_RUNTIME}"
35+
echo "job: ${JOB_NAME}"
36+
echo "test home: ${TEST_HOME}"
37+
echo "kernel: $(uname -v)"
38+
echo "uptime: $(uptime)"
39+
# Setting KUBECONFIG prevents the version check from erroring out due to permission issues
40+
echo "kubectl: $(env KUBECONFIG=${TEST_HOME} kubectl version --client)"
41+
echo "docker: $(docker version --format '{{ .Client.Version }}')"
42+
echo "podman: $(sudo podman version --format '{{.Version}}' || true)"
43+
echo "go: $(go version || true)"
44+
45+
case "${DRIVER}" in
46+
kvm2)
47+
echo "virsh: $(virsh --version)"
48+
;;
49+
virtualbox)
50+
echo "vbox: $(vboxmanage --version)"
51+
;;
52+
vfkit)
53+
echo "vfkit: $(vfkit --version)"
54+
;;
55+
krunkit)
56+
echo "krunkit: $(krunkit --version)"
57+
;;
58+
esac
59+
60+
echo ""
61+
}
62+
63+
function install_dependencies() {
64+
# We need pstree for the restart cronjobs
65+
if [ "$(uname)" != "Darwin" ]; then
66+
sudo apt-get -y install lsof psmisc dnsutils
67+
else
68+
brew install pstree coreutils pidof
69+
ln -s /usr/local/bin/gtimeout /usr/local/bin/timeout || true
70+
fi
71+
# install golang if not present
72+
sudo hack/prow/installer/check_install_golang.sh /usr/local 1.24.5 || true
73+
# install gotestsum if not present
74+
GOROOT="/usr/local/go" hack/prow/installer/check_install_gotestsum.sh || true
75+
# install gopogh
76+
hack/prow/installer/check_install_gopogh.sh || true
77+
78+
# install jq
79+
if ! type "jq" >/dev/null; then
80+
echo ">> Installing jq"
81+
if [ "${ARCH}" == "arm64" && "${OS}" == "linux" ]; then
82+
sudo apt-get install jq -y
83+
elif [ "${ARCH}" == "arm64" ]; then
84+
echo "Unable to install 'jq' automatically for arm64 on Darwin, please install 'jq' manually."
85+
exit 5
86+
elif [ "${OS}" != "darwin" ]; then
87+
curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 && sudo install jq-linux64 /usr/local/bin/jq
88+
else
89+
curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 && sudo install jq-osx-amd64 /usr/local/bin/jq
90+
fi
91+
fi
92+
93+
}
94+
95+
function docker_setup() {
96+
97+
# clean all docker artifacts up
98+
docker system prune -a --volumes -f || true
99+
docker system df || true
100+
docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
101+
102+
# read only token, never expires
103+
#todo: do we need this token
104+
# docker login -u minikubebot -p "$DOCKERHUB_READONLY_TOKEN"
105+
}
106+
107+
function gvisor_image_build() {
108+
# Build the gvisor image so that we can integration test changes to pkg/gvisor
109+
chmod +x testdata/gvisor-addon
110+
# skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137
111+
if [ "$(uname)" != "Darwin" ]; then
112+
# Should match GVISOR_IMAGE_VERSION in Makefile
113+
docker build -t gcr.io/k8s-minikube/gvisor-addon:2 -f testdata/gvisor-addon-Dockerfile ./testdata
114+
fi
115+
}
116+
117+
function run_gopogh() {
118+
# todo: currently we do not save to gopogh db
119+
echo "Not saving to DB"
120+
gopogh -in "${JSON_OUT}" -out_html "${HTML_OUT}" -out_summary "${SUMMARY_OUT}" -name "${JOB_NAME}" -pr "${PULL_NUMBER}" -repo github.com/kubernetes/minikube/ -details "${COMMIT}:$(date +%Y-%m-%d)"
121+
122+
}
123+
124+
# this is where the script starts
125+
readonly OS_ARCH="${OS}-${ARCH}"
126+
readonly TEST_ROOT="${HOME}/minikube-integration"
127+
readonly TEST_HOME="${TEST_ROOT}/${MINIKUBE_LOCATION}-$$"
128+
129+
export GOPATH="$HOME/go"
130+
export KUBECONFIG="${TEST_HOME}/kubeconfig"
131+
export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
132+
export MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
133+
134+
readonly TIMEOUT=120m
135+
136+
cp -r test/integration/testdata .
137+
138+
# Add the out/ directory to the PATH, for using new drivers.
139+
export PATH="$(pwd)/out/":$PATH
140+
mkdir -p "${TEST_ROOT}"
141+
mkdir -p "${TEST_HOME}"
142+
export MINIKUBE_HOME="${TEST_HOME}/.minikube"
143+
export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
144+
export E2E_BIN="out/e2e-${OS_ARCH}"
145+
146+
install_dependencies
147+
docker_setup
148+
149+
150+
if [ "$CONTAINER_RUNTIME" == "containerd" ]; then
151+
cp out/gvisor-addon testdata/
152+
gvisor_image_build
153+
fi
154+
155+
print_test_info
156+
157+
readonly TEST_OUT="${TEST_HOME}/testout.txt"
158+
readonly JSON_OUT="${TEST_HOME}/test.json"
159+
readonly JUNIT_OUT="${TEST_HOME}/junit-unit.xml"
160+
readonly HTML_OUT="${TEST_HOME}/test.html"
161+
readonly SUMMARY_OUT="${TEST_HOME}/test_summary.json"
162+
163+
touch "${TEST_OUT}"
164+
touch "${JSON_OUT}"
165+
touch "${JUNIT_OUT}"
166+
touch "${HTML_OUT}"
167+
touch "${SUMMARY_OUT}"
168+
169+
e2e_start_time="$(date -u +%s)"
170+
echo ""
171+
echo ">> Starting ${E2E_BIN} at $(date)"
172+
set -x
173+
174+
EXTRA_START_ARGS="${EXTRA_START_ARGS} --container-runtime=${CONTAINER_RUNTIME}"
175+
echo $PATH
176+
gotestsum --jsonfile "${JSON_OUT}" --junitfile="${JUNIT_OUT}" -f standard-verbose --raw-command -- \
177+
go tool test2json -t \
178+
${E2E_BIN} \
179+
-minikube-start-args="--driver=${DRIVER} ${EXTRA_START_ARGS}" \
180+
-test.timeout=${TIMEOUT} -test.v \
181+
${EXTRA_TEST_ARGS} \
182+
-binary="${MINIKUBE_BIN}" 2>&1 |
183+
tee "${TEST_OUT}"
184+
185+
result=${PIPESTATUS[0]} # capture the exit code of the first cmd in pipe.
186+
set +x
187+
echo ">> ${E2E_BIN} exited with ${result} at $(date)"
188+
echo ""
189+
190+
# calculate the time took to finish running e2e binary test.
191+
e2e_end_time="$(date -u +%s)"
192+
elapsed=$(($e2e_end_time - $e2e_start_time))
193+
min=$(($elapsed / 60))
194+
sec=$(tail -c 3 <<<$((${elapsed}00 / 60)))
195+
elapsed=$min.$sec
196+
197+
#todo: currently we skip gopogh upload , we shall add it back
198+
run_gopogh
199+
200+
# according to prow's requirement, upload the test report to $ARTIFACTS
201+
cp ${TEST_OUT} .
202+
cp ${JSON_OUT} .
203+
cp ${JUNIT_OUT} .
204+
cp ${HTML_OUT} .
205+
cp ${SUMMARY_OUT} .
206+
if [[ $result -eq 0 ]]; then
207+
echo "minikube: SUCCESS"
208+
else
209+
echo "minikube: FAIL"
210+
fi
211+
212+
exit "$result"

‎hack/prow/docker.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"Image": "debian"
3+
}

‎hack/prow/installer/check_install_docker.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
# Copyright 2016 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -eux -o pipefail
18+
19+
ARCH=${ARCH:=amd64}
20+
21+
22+
echo "Installing latest docker"
23+
curl -fsSL https://get.docker.com -o get-docker.sh
24+
sudo sh get-docker.sh
25+
rm get-docker.sh
26+
27+
sudo usermod -aG docker minitest || true
28+
29+
echo "Installing latest kubectl"
30+
curl -LO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
31+
sudo install ./kubectl /usr/local/bin/kubectl

‎hack/prow/installer/check_install_golang.sh‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/bin/bash
2+
3+
# Copyright 2025 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# This script requires two parameters:
18+
# $1. INSTALL_PATH: The path to install the golang binary
19+
# $2. GO_VERSION: The version of golang to install
20+
21+
22+
23+
set -eux -o pipefail
24+
25+
if (($# < 2)); then
26+
echo "ERROR: given ! ($#) parameters but expected 2."
27+
echo "USAGE: ./check_install_golang.sh INSTALL_PATH GO_VERSION"
28+
exit 1
29+
fi
30+
31+
VERSION_TO_INSTALL=${2}
32+
INSTALL_PATH=${1}
33+
34+
function current_arch() {
35+
case $(arch) in
36+
"x86_64" | "i386")
37+
echo "amd64"
38+
;;
39+
"aarch64" | "arm64")
40+
echo "arm64"
41+
;;
42+
*)
43+
echo "unexpected arch: $(arch). use amd64" 1>&2
44+
echo "amd64"
45+
;;
46+
esac
47+
}
48+
49+
ARCH=${ARCH:=$(current_arch)}
50+
51+
# installs or updates golang if right version doesn't exists
52+
function check_and_install_golang() {
53+
if ! go version &>/dev/null; then
54+
echo "WARNING: No golang installation found in your environment."
55+
install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
56+
return
57+
fi
58+
59+
sudo chown -R $USER:$USER "$INSTALL_PATH"/go
60+
61+
# golang has been installed and check its version
62+
if [[ $(go version | cut -d' ' -f 3) =~ go(([0-9]+)\.([0-9]+).([0-9]+)*) ]]; then
63+
HOST_VERSION=${BASH_REMATCH[1]}
64+
if [ $HOST_VERSION == $VERSION_TO_INSTALL ]; then
65+
echo "go version on the host looks good : $HOST_VERSION"
66+
else
67+
echo "WARNING: expected go version to be $VERSION_TO_INSTALL but got $HOST_VERSION"
68+
install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
69+
fi
70+
else
71+
echo "ERROR: Failed to parse golang version: $(go version)"
72+
return
73+
fi
74+
}
75+
76+
# install_golang takes two parameters version and path to install.
77+
function install_golang() {
78+
local -r GO_VER="$1"
79+
local -r GO_DIR="$2/go"
80+
echo "Installing golang version: $GO_VER in $GO_DIR"
81+
82+
INSTALLOS=linux
83+
if [[ "$OSTYPE" == "darwin"* ]]; then
84+
INSTALLOS=darwin
85+
fi
86+
87+
local -r GO_TGZ="go${GO_VER}.${INSTALLOS}-${ARCH}.tar.gz"
88+
pushd /tmp
89+
90+
# using sudo because previously installed versions might have been installed by a different user.
91+
sudo rm -rf "$GO_TGZ"
92+
curl -qL -O "https://storage.googleapis.com/golang/$GO_TGZ"
93+
sudo rm -rf "$GO_DIR"
94+
sudo mkdir -p "$GO_DIR"
95+
sudo tar -C "$GO_DIR" --strip-components=1 -xzf "$GO_TGZ"
96+
97+
popd >/dev/null
98+
echo "installed in $GO_DIR: $($GO_DIR/bin/go version)"
99+
}
100+
101+
check_and_install_golang

‎hack/prow/installer/check_install_gopogh.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/bash
2+
3+
# Copyright 2025 The Kubernetes Authors All rights reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# this script requires golang to be installed
18+
19+
set -eux -o pipefail
20+
21+
go install github.com/medyagh/gopogh/cmd/[email protected]

0 commit comments

Comments
 (0)