Skip to content

Commit 543c84e

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #786 from BenTheElder/no-kubetest-shrug
refactor e2e.sh
2 parents 27a4b6c + 97b044c commit 543c84e

File tree

1 file changed

+113
-148
lines changed

1 file changed

+113
-148
lines changed

‎hack/ci/e2e.sh‎

Lines changed: 113 additions & 148 deletions
Original file line numberDiff line numberDiff line change
@@ -15,89 +15,64 @@
1515

1616
# hack script for running a kind e2e
1717
# must be run with a kubernetes checkout in $PWD (IE from the checkout)
18-
# TODO(bentheelder): replace this with kubetest integration
1918
# Usage: SKIP="ginkgo skip regex" FOCUS="ginkgo focus regex" kind-e2e.sh
2019

21-
set -o errexit
22-
set -o nounset
23-
set -o pipefail
24-
set -o xtrace
20+
set -o errexit -o nounset -o pipefail -o xtrace
2521

2622
# our exit handler (trap)
2723
cleanup() {
28-
# always attempt to dump logs
29-
kind "export" logs "${ARTIFACTS}/logs" || true
30-
# KIND_IS_UP is true once we: kind create
31-
if [[ "${KIND_IS_UP:-}" = true ]]; then
32-
kind delete cluster || true
33-
fi
34-
# clean up e2e.test symlink
35-
rm -f _output/bin/e2e.test
36-
# remove our tempdir
37-
# NOTE: this needs to be last, or it will prevent kind delete
38-
if [[ -n "${TMP_DIR:-}" ]]; then
39-
rm -rf "${TMP_DIR}"
40-
fi
24+
# always attempt to dump logs
25+
kind "export" logs "${ARTIFACTS}/logs" || true
26+
# KIND_IS_UP is true once we: kind create
27+
if [[ "${KIND_IS_UP:-}" = true ]]; then
28+
kind delete cluster || true
29+
fi
30+
# clean up e2e.test symlink
31+
rm -f _output/bin/e2e.test || true
32+
# remove our tempdir
33+
# NOTE: this needs to be last, or it will prevent kind delete
34+
if [[ -n "${TMP_DIR:-}" ]]; then
35+
rm -rf "${TMP_DIR}"
36+
fi
4137
}
4238

4339
# install kind to a tempdir GOPATH from this script's kind checkout
4440
install_kind() {
45-
# install `kind` to tempdir
46-
TMP_DIR=$(mktemp -d)
47-
# ensure bin dir
48-
mkdir -p "${TMP_DIR}/bin"
49-
# install
50-
local script_dir
51-
script_dir="$(dirname "${BASH_SOURCE[0]}")"
52-
make -C "${script_dir}/../.." install INSTALL_PATH="${TMP_DIR}/bin"
53-
# ensure it is in path
54-
PATH="${TMP_DIR}/bin:${PATH}"
55-
export PATH
41+
mkdir -p "${TMP_DIR}/bin"
42+
make -C "$(dirname "${BASH_SOURCE[0]}")/../.." install INSTALL_PATH="${TMP_DIR}/bin"
43+
export PATH="${TMP_DIR}/bin:${PATH}"
5644
}
5745

5846
# build kubernetes / node image, e2e binaries
5947
build() {
60-
# possibly enable bazel build caching before building kubernetes
61-
BAZEL_REMOTE_CACHE_ENABLED=${BAZEL_REMOTE_CACHE_ENABLED:-false}
62-
if [[ "${BAZEL_REMOTE_CACHE_ENABLED}" == "true" ]]; then
63-
# run the script in the kubekins image, do not fail if it fails
64-
/usr/local/bin/create_bazel_cache_rcs.sh || true
65-
fi
66-
67-
# build the node image w/ kubernetes
68-
# TODO(bentheelder): remove the kube-root flag after we make kind try to
69-
# find this in a go module compatible way
70-
kind build node-image --type=bazel \
71-
--kube-root="$(go env GOPATH)/src/k8s.io/kubernetes"
72-
73-
# make sure we have e2e requirements
74-
#make all WHAT="cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo"
75-
bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo
76-
77-
# e2e.test does not show up in a path with platform in it and will not be found
78-
# by kube::util::find-binary, so we will copy it to an acceptable location
79-
# until this is fixed upstream
80-
# https://github.com/kubernetes/kubernetes/issues/68306
81-
mkdir -p "_output/bin/"
82-
cp "bazel-bin/test/e2e/e2e.test" "_output/bin/"
83-
84-
# try to make sure the kubectl we built is in PATH
85-
local maybe_kubectl
86-
maybe_kubectl="$(find "${PWD}/bazel-bin/" -name "kubectl" -type f)"
87-
if [[ -n "${maybe_kubectl}" ]]; then
88-
PATH="$(dirname "${maybe_kubectl}"):${PATH}"
89-
export PATH
90-
fi
91-
92-
# release some memory after building
93-
sync || true
94-
echo 1 > /proc/sys/vm/drop_caches || true
48+
# possibly enable bazel build caching before building kubernetes
49+
if [[ "${BAZEL_REMOTE_CACHE_ENABLED:-false}" == "true" ]]; then
50+
create_bazel_cache_rcs.sh || true
51+
fi
52+
53+
# build the node image w/ kubernetes
54+
kind build node-image --type=bazel --kube-root="$(go env GOPATH)/src/k8s.io/kubernetes"
55+
56+
# make sure we have e2e requirements
57+
#make all WHAT="cmd/kubectl test/e2e/e2e.test vendor/github.com/onsi/ginkgo/ginkgo"
58+
bazel build //cmd/kubectl //test/e2e:e2e.test //vendor/github.com/onsi/ginkgo/ginkgo
59+
60+
# ensure the e2e script will find our binaries ...
61+
# https://github.com/kubernetes/kubernetes/issues/68306
62+
mkdir -p '_output/bin/'
63+
cp "bazel-bin/test/e2e/e2e.test" "_output/bin/"
64+
PATH="$(dirname "$(find "${PWD}/bazel-bin/" -name kubectl -type f)"):${PATH}"
65+
export PATH
66+
67+
# attempt to release some memory after building
68+
sync || true
69+
echo 1 > /proc/sys/vm/drop_caches || true
9570
}
9671

9772
# up a cluster with kind
9873
create_cluster() {
99-
# create the config file
100-
cat <<EOF > "${ARTIFACTS}/kind-config.yaml"
74+
# create the config file
75+
cat <<EOF > "${ARTIFACTS}/kind-config.yaml"
10176
# config for 1 control plane node and 2 workers
10277
# necessary for conformance
10378
kind: Cluster
@@ -111,49 +86,43 @@ nodes:
11186
- role: worker
11287
EOF
11388

114-
# mark the cluster as up for cleanup
115-
# even if kind create fails, kind delete can clean up after it
116-
KIND_IS_UP=true
117-
# actually create, with:
118-
# - do not delete created nodes from a failed cluster create (for debugging)
119-
# - wait up to one minute for the nodes to be "READY"
120-
# - set log leve to debug
121-
# - use our multi node config
122-
kind create cluster \
123-
--image=kindest/node:latest \
124-
--retain \
125-
--wait=1m \
126-
--loglevel=debug \
127-
"--config=${ARTIFACTS}/kind-config.yaml"
89+
# actually create the cluster
90+
KIND_IS_UP=true
91+
kind create cluster \
92+
--image=kindest/node:latest \
93+
--retain \
94+
--wait=1m \
95+
--loglevel=debug \
96+
"--config=${ARTIFACTS}/kind-config.yaml"
12897
}
12998

13099
# run e2es with kubetest
131100
run_tests() {
132-
# export the KUBECONFIG
133-
KUBECONFIG="$(kind get kubeconfig-path)"
134-
export KUBECONFIG
135-
136-
if [[ "${IP_FAMILY:-ipv4}" == "ipv6" ]]; then
137-
# IPv6 clusters need some CoreDNS changes in order to work in k8s CI:
138-
# 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
139-
# to work in an offline environment:
140-
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
141-
# 2. k8s CI adds following domains to resolv.conf search field :
142-
# c.k8s-prow-builds.internal google.internal.
143-
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
144-
# otherwise pods stops trying to resolve the domain.
145-
# The difference against the default CoreDNS config in k8s 1.15 is:
146-
# < kubernetes cluster.local in-addr.arpa ip6.arpa {
147-
# ---
148-
# > kubernetes cluster.local internal in-addr.arpa ip6.arpa {
149-
# 9,10d9
150-
# < upstream
151-
# < fallthrough in-addr.arpa ip6.arpa
152-
# 13,15d11
153-
# < forward . /etc/resolv.conf
154-
# < loop
155-
# 21c17,20
156-
cat <<EOF | kubectl apply -f -
101+
# export the KUBECONFIG
102+
KUBECONFIG="$(kind get kubeconfig-path)"
103+
export KUBECONFIG
104+
105+
if [[ "${IP_FAMILY:-ipv4}" == "ipv6" ]]; then
106+
# IPv6 clusters need some CoreDNS changes in order to work in k8s CI:
107+
# 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
108+
# to work in an offline environment:
109+
# https://github.com/coredns/coredns/issues/2494#issuecomment-457215452
110+
# 2. k8s CI adds following domains to resolv.conf search field :
111+
# c.k8s-prow-builds.internal google.internal.
112+
# CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL
113+
# otherwise pods stops trying to resolve the domain.
114+
# The difference against the default CoreDNS config in k8s 1.15 is:
115+
# < kubernetes cluster.local in-addr.arpa ip6.arpa {
116+
# ---
117+
# > kubernetes cluster.local internal in-addr.arpa ip6.arpa {
118+
# 9,10d9
119+
# < upstream
120+
# < fallthrough in-addr.arpa ip6.arpa
121+
# 13,15d11
122+
# < forward . /etc/resolv.conf
123+
# < loop
124+
# 21c17,20
125+
cat <<EOF | kubectl apply -f -
157126
---
158127
apiVersion: v1
159128
data:
@@ -175,54 +144,50 @@ metadata:
175144
namespace: kube-system
176145
---
177146
EOF
147+
fi
148+
149+
# ginkgo regexes
150+
SKIP="${SKIP:-}"
151+
FOCUS="${FOCUS:-"\\[Conformance\\]"}"
152+
# if we set PARALLEL=true, skip serial tests set --ginkgo-parallel
153+
if [[ "${PARALLEL:-false}" == "true" ]]; then
154+
export GINKGO_PARALLEL=y
155+
if [[ -z "${SKIP}" ]]; then
156+
SKIP="\\[Serial\\]"
157+
else
158+
SKIP="\\[Serial\\]|${SKIP}"
178159
fi
179-
180-
# base kubetest args
181-
KUBETEST_ARGS="--provider=skeleton --test --check-version-skew=false"
182-
183-
# get the number of worker nodes
184-
# TODO(bentheelder): this is kinda gross
185-
NUM_NODES="$(kubectl get nodes \
186-
-o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \
187-
| grep -cv "node-role.kubernetes.io/master" \
188-
)"
189-
190-
# ginkgo regexes
191-
SKIP="${SKIP:-}"
192-
FOCUS="${FOCUS:-"\\[Conformance\\]"}"
193-
# if we set PARALLEL=true, skip serial tests set --ginkgo-parallel
194-
PARALLEL="${PARALLEL:-false}"
195-
if [[ "${PARALLEL}" == "true" ]]; then
196-
if [[ -z "${SKIP}" ]]; then
197-
SKIP="\\[Serial\\]"
198-
else
199-
SKIP="\\[Serial\\]|${SKIP}"
200-
fi
201-
KUBETEST_ARGS="${KUBETEST_ARGS} --ginkgo-parallel"
202-
fi
203-
204-
# add ginkgo args
205-
KUBETEST_ARGS="${KUBETEST_ARGS} --test_args=\"--ginkgo.focus=${FOCUS} --ginkgo.skip=${SKIP} --report-dir=${ARTIFACTS} --disable-log-dump=true --num-nodes=${NUM_NODES}\""
206-
207-
# setting this env prevents ginkg e2e from trying to run provider setup
208-
export KUBERNETES_CONFORMANCE_TEST="y"
209-
210-
# run kubetest, if it fails clean up and exit failure
211-
eval "kubetest ${KUBETEST_ARGS}"
160+
fi
161+
162+
# get the number of worker nodes
163+
# TODO(bentheelder): this is kinda gross
164+
NUM_NODES="$(kubectl get nodes \
165+
-o=jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.taints}{"\n"}{end}' \
166+
| grep -cv "node-role.kubernetes.io/master" )"
167+
168+
# setting this env prevents ginkg e2e from trying to run provider setup
169+
export KUBERNETES_CONFORMANCE_TEST="y"
170+
# run the tests
171+
./hack/ginkgo-e2e.sh \
172+
'--provider=skeleton' "--num-nodes=${NUM_NODES}" \
173+
"--ginkgo.focus=${FOCUS}" "--ginkgo.skip=${SKIP}" \
174+
"--report-dir=${ARTIFACTS}" '--disable-log-dump=true'
212175
}
213176

214177
# setup kind, build kubernetes, create a cluster, run the e2es
215178
main() {
216-
# ensure artifacts exists when not in CI
217-
ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
218-
mkdir -p "${ARTIFACTS}"
219-
export ARTIFACTS
220-
# now build an run the cluster and tests
221-
trap cleanup EXIT
222-
install_kind
223-
build
224-
create_cluster
225-
run_tests
179+
# create temp dir and setup cleanup
180+
TMP_DIR=$(mktemp -d)
181+
trap cleanup EXIT
182+
# ensure artifacts exists when not in CI
183+
ARTIFACTS="${ARTIFACTS:-${PWD}/_artifacts}"
184+
export ARTIFACTS
185+
mkdir -p "${ARTIFACTS}"
186+
# now build and run the cluster and tests
187+
install_kind
188+
build
189+
create_cluster
190+
run_tests
226191
}
227192

228193
main

0 commit comments

Comments
 (0)