diff --git a/contrib/dind/kubespray-dind.yaml b/contrib/dind/kubespray-dind.yaml
index 628ed787b2df22be1b600ad075b3b246df221d91..02cd16989418813adab5213307c960a1d8620142 100644
--- a/contrib/dind/kubespray-dind.yaml
+++ b/contrib/dind/kubespray-dind.yaml
@@ -15,8 +15,6 @@ docker_storage_options: -s overlay2 --storage-opt overlay2.override_kernel_check
dns_mode: coredns
-kube_network_plugin: weave
-
deploy_netchecker: True
netcheck_agent_image_repo: quay.io/l23network/k8s-netchecker-agent
netcheck_server_image_repo: quay.io/l23network/k8s-netchecker-server
diff --git a/contrib/dind/run-test-distros.sh b/contrib/dind/run-test-distros.sh
old mode 100644
new mode 100755
index e1c2c3ef7998a37463a02c5e7fb4494dcc0078e8..6c2d4cbc61c63c9865c6550e48d3bb913e28150a
--- a/contrib/dind/run-test-distros.sh
+++ b/contrib/dind/run-test-distros.sh
@@ -2,45 +2,89 @@
# Q&D test'em all: creates full DIND kubespray deploys
# for each distro, verifying it via netchecker.
+info() {
+ local msg="$*"
+ local date="$(date -Isec)"
+ echo "INFO: [$date] $msg"
+}
pass_or_fail() {
local rc="$?"
local msg="$*"
local date="$(date -Isec)"
[ $rc -eq 0 ] && echo "PASS: [$date] $msg" || echo "FAIL: [$date] $msg"
+ return $rc
}
test_distro() {
- local distro=${1:?}
- ansible-playbook -i hosts dind-cluster.yaml --extra-vars node_distro=$distro
- pass_or_fail "$distro: dind-nodes"
+ local distro=${1:?};shift
+ local extra="${*:-}"
+ local prefix="$distro[${extra}]}"
+ ansible-playbook -i hosts dind-cluster.yaml -e node_distro=$distro
+ pass_or_fail "$prefix: dind-nodes" || return 1
(cd ../..
INVENTORY_DIR=inventory/local-dind
mkdir -p ${INVENTORY_DIR}
rm -f ${INVENTORY_DIR}/hosts.ini
CONFIG_FILE=${INVENTORY_DIR}/hosts.ini /tmp/kubespray.dind.inventory_builder.sh
+ # expand $extra with -e in front of each word
+ extra_args=""; for extra_arg in $extra; do extra_args="$extra_args -e $extra_arg"; done
ansible-playbook --become -e ansible_ssh_user=$distro -i \
${INVENTORY_DIR}/hosts.ini cluster.yml \
- --extra-vars @contrib/dind/kubespray-dind.yaml --extra-vars bootstrap_os=$distro
- pass_or_fail "$distro: kubespray"
- )
- docker exec kube-node1 kubectl get pod --all-namespaces
- pass_or_fail "$distro: kube-api"
- let n=60
- while ((n--)); do
- docker exec kube-node1 curl -s http://localhost:31081/api/v1/connectivity_check | grep successfully && break
+ -e @contrib/dind/kubespray-dind.yaml -e bootstrap_os=$distro ${extra_args}
+ pass_or_fail "$prefix: kubespray"
+ ) || return 1
+ local node0=${NODES[0]}
+ docker exec ${node0} kubectl get pod --all-namespaces
+ pass_or_fail "$prefix: kube-api" || return 1
+ let retries=60
+ while ((retries--)); do
+ # Some CNI may set NodePort on "main" node interface address (thus no localhost NodePort)
+ # e.g. kube-router: https://github.com/cloudnativelabs/kube-router/pull/217
+ docker exec ${node0} curl -m2 -s http://${NETCHECKER_HOST:?}:31081/api/v1/connectivity_check | grep successfully && break
sleep 2
done
- [ $n -ge 0 ]
- pass_or_fail "$distro: netcheck"
+ [ $retries -ge 0 ]
+ pass_or_fail "$prefix: netcheck" || return 1
}
-# Get all DISTROS from distro.yaml if $* unset (shame no yaml parsing, but nuff anyway)
-DISTROS="${*:-$(egrep -o '^ \w+' group_vars/all/distro.yaml|paste -s)}"
-NODES="$(egrep ^kube-node hosts|paste -s)"
-echo "DISTROS=${DISTROS}"
-for distro in ${DISTROS}; do
- docker rm -f ${NODES}
- { time test_distro ${distro} ;} |& tee test-${distro}.out
- # sleeping for the sake of the human to verify if they want
- sleep 2m
+NODES=($(egrep ^kube-node hosts))
+NETCHECKER_HOST=localhost
+
+: ${OUTPUT_DIR:=./out}
+mkdir -p ${OUTPUT_DIR}
+
+# We expect the SPEC file(s) to have below two arrays as e.g.
+# DISTROS=(debian centos)
+# EXTRAS=(
+# 'kube_network_plugin=calico'
+# 'kube_network_plugin=flannel'
+# 'kube_network_plugin=weave'
+# )
+# that we'll test in a "combinatory" way (e.g. above will be 6 tests),
+# creating a sequenced <spec_filename>-nn.out with each output
+
+SPECS=${*:?Missing SPEC files, e.g. test-most_distros-some_CNIs.env}
+for spec in ${SPECS}; do
+ unset DISTROS EXTRAS
+ echo "Loading file=${spec} ..."
+ . ${spec} || continue
+ : ${DISTROS:?} || continue
+ echo "DISTROS=${DISTROS[@]}"
+ echo "EXTRAS->"
+ printf " %s\n" "${EXTRAS[@]}"
+ let n=1
+ for distro in ${DISTROS[@]}; do
+ for extra in "${EXTRAS[@]:-NULL}"; do
+ # Magic value to let this for run once:
+ [[ ${extra} == NULL ]] && unset extra
+ docker rm -f ${NODES[@]}
+ printf -v file_out "%s/%s-%02d.out" ${OUTPUT_DIR} ${spec} $((n++))
+ {
+ info "${distro}[${extra}] START: file_out=${file_out}"
+ time test_distro ${distro} ${extra}
+ } |& tee ${file_out}
+ # sleeping for the sake of the human to verify if they want
+ sleep 2m
+ done
+ done
done
-egrep '^(PASS:|FAIL:|real)' $(ls -tr test-*out)
+egrep -H '^(....:|real)' $(ls -tr ${OUTPUT_DIR}/*.out)
diff --git a/contrib/dind/test-most_distros-some_CNIs.env b/contrib/dind/test-most_distros-some_CNIs.env
new file mode 100644
index 0000000000000000000000000000000000000000..f6e4e1a8ca320793ca995089f9c537bc8a3f1453
--- /dev/null
+++ b/contrib/dind/test-most_distros-some_CNIs.env
@@ -0,0 +1,11 @@
+# Test spec file: used from ./run-test-distros.sh, will run
+# each distro in $DISTROS overloading main kubespray ansible-playbook run
+# Get all DISTROS from distro.yaml (shame no yaml parsing, but nuff anyway)
+# DISTROS="${*:-$(egrep -o '^ \w+' group_vars/all/distro.yaml|paste -s)}"
+DISTROS=(debian ubuntu centos fedora)
+
+# Each line below will be added as --extra-vars to main playbook run
+EXTRAS=(
+ 'kube_network_plugin=calico'
+ 'kube_network_plugin=weave'
+)
diff --git a/contrib/dind/test-some_distros-kube_router_combo.env b/contrib/dind/test-some_distros-kube_router_combo.env
new file mode 100644
index 0000000000000000000000000000000000000000..4b0767b96e9f3feab76b94cf48f9a110b210abd9
--- /dev/null
+++ b/contrib/dind/test-some_distros-kube_router_combo.env
@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+NETCHECKER_HOST=${NODES[0]}
+EXTRAS=(
+ 'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":false}'
+ 'kube_network_plugin=kube-router {"kubeadm_enabled":true,"kube_router_run_service_proxy":true}'
+ 'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":false}'
+ 'kube_network_plugin=kube-router {"kubeadm_enabled":false,"kube_router_run_service_proxy":true}'
+)
diff --git a/contrib/dind/test-some_distros-most_CNIs.env b/contrib/dind/test-some_distros-most_CNIs.env
new file mode 100644
index 0000000000000000000000000000000000000000..830695e5f71190ac75cb428ad22904bd6f3ad22c
--- /dev/null
+++ b/contrib/dind/test-some_distros-most_CNIs.env
@@ -0,0 +1,8 @@
+DISTROS=(debian centos)
+EXTRAS=(
+ 'kube_network_plugin=calico {"kubeadm_enabled":true}'
+ 'kube_network_plugin=canal {"kubeadm_enabled":true}'
+ 'kube_network_plugin=cilium {"kubeadm_enabled":true}'
+ 'kube_network_plugin=flannel {"kubeadm_enabled":true}'
+ 'kube_network_plugin=weave {"kubeadm_enabled":true}'
+)