From ad68b23d8a6bbb39892570855768a9c6a63dca93 Mon Sep 17 00:00:00 2001
From: Bogdan Dobrelya <bdobrelia@mirantis.com>
Date: Wed, 14 Dec 2016 17:00:38 +0100
Subject: [PATCH] Manual steps for Gitlab CI pipeline
* Reduce default testcase to 2 nodes, add HA case.
* Adjust gen_matrix script for Travis/Gitlab CIs.
* Enable netchecker deploy foro gitlab CI.
* Sync other things from travis matrix and reorder them as build steps
for pull requests, master branch, auto/manual.
* Do auto-step1 from part1 and manual step2,3 for branches/PRs.
* Do manual steps from part2, special for master merges.
Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>
---
.gitlab-ci.yml | 84 +++++++++++++++-------------
.travis.yml.bak | 15 ++---
docs/test_cases.md | 37 +++++++++---
scripts/gen_matrix.sh | 15 ++++-
tests/cloud_playbooks/create-gce.yml | 11 +++-
tests/cloud_playbooks/delete-gce.yml | 11 +++-
tests/templates/inventory-gce.j2 | 15 ++++-
7 files changed, 128 insertions(+), 60 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 69e646a67..ef60e447e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,7 +2,7 @@ stages:
- unit-tests
- deploy-gce-part1
- deploy-gce-part2
- - deploy-gce-separated
+ - deploy-gce-special
variables:
FAILFASTCI_NAMESPACE: 'kargo-ci'
@@ -101,6 +101,7 @@ before_script:
-e ansible_python_interpreter=${PYPATH}
-e download_run_once=true
-e download_localhost=true
+ -e deploy_netchecker=true
-e local_release_dir=${PWD}/downloads
cluster.yml
@@ -109,15 +110,16 @@ before_script:
## Test Master API
- ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL
- ## Create a POD
- - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/020_check-create-pod.yml $LOG_LEVEL
-
## Ping the between 2 pod
- ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+ ## Advanced DNS checks
+ - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+
after_script:
- >
ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL
+ -e mode=${CLUSTER_MODE}
-e test_id=${TEST_ID}
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-e gce_project_id=${GCE_PROJECT_ID}
@@ -127,29 +129,32 @@ before_script:
-e inventory_path=${PWD}/inventory/inventory.ini
-e cloud_region=${CLOUD_REGION}
-debian8-flannel:
+coreos-calico:
stage: deploy-gce-part1
<<: *job
<<: *gce
variables:
<<: *gce_variables
- KUBE_NETWORK_PLUGIN: canal
- CLOUD_IMAGE: debian-8-kubespray
- CLOUD_REGION: us-west1-a
- CLUSTER_MODE: default
-
+ KUBE_NETWORK_PLUGIN: calico
+ CLOUD_IMAGE: coreos-stable
+ CLOUD_REGION: us-west1-b
+ CLUSTER_MODE: separated
+ BOOTSTRAP_OS: coreos
+ when: on_success
+ #only: [/^pr-.*$/]
-coreos-canal:
+debian8-canal:
stage: deploy-gce-part1
<<: *job
<<: *gce
variables:
<<: *gce_variables
KUBE_NETWORK_PLUGIN: canal
- CLOUD_IMAGE: coreos-stable
+ CLOUD_IMAGE: debian-8-kubespray
CLOUD_REGION: us-east1-b
- CLUSTER_MODE: default
- BOOTSTRAP_OS: coreos
+ CLUSTER_MODE: ha
+ when: manual
+ #only: [/^pr-.*$/]
rhel7-weave:
stage: deploy-gce-part1
@@ -161,6 +166,8 @@ rhel7-weave:
CLOUD_IMAGE: rhel-7
CLOUD_REGION: europe-west1-b
CLUSTER_MODE: default
+ when: manual
+ #only: [/^pr-.*$/]
centos7-flannel:
stage: deploy-gce-part2
@@ -172,8 +179,8 @@ centos7-flannel:
CLOUD_IMAGE: centos-7
CLOUD_REGION: us-west1-a
CLUSTER_MODE: default
- only:
- - master
+ when: manual
+ only: ['master']
debian8-calico:
stage: deploy-gce-part2
@@ -185,24 +192,25 @@ debian8-calico:
CLOUD_IMAGE: debian-8-kubespray
CLOUD_REGION: us-central1-b
CLUSTER_MODE: default
- only:
- - master
+ when: manual
+ only: ['master']
-centos7-calico:
+coreos-canal:
stage: deploy-gce-part2
<<: *job
<<: *gce
variables:
<<: *gce_variables
- KUBE_NETWORK_PLUGIN: calico
- CLOUD_IMAGE: centos-7
- CLOUD_REGION: europe-west1-b
+ KUBE_NETWORK_PLUGIN: canal
+ CLOUD_IMAGE: coreos-stable
+ CLOUD_REGION: us-east1-b
CLUSTER_MODE: default
- only:
- - master
+ BOOTSTRAP_OS: coreos
+ when: manual
+ only: ['master']
rhel7-canal:
- stage: deploy-gce-separated
+ stage: deploy-gce-special
<<: *job
<<: *gce
variables:
@@ -211,11 +219,11 @@ rhel7-canal:
CLOUD_IMAGE: rhel-7
CLOUD_REGION: us-east1-b
CLUSTER_MODE: separated
- only:
- - master
+ when: manual
+ only: ['master']
ubuntu-weave:
- stage: deploy-gce-separated
+ stage: deploy-gce-special
<<: *job
<<: *gce
variables:
@@ -224,23 +232,21 @@ ubuntu-weave:
CLOUD_IMAGE: ubuntu-1604-xenial
CLOUD_REGION: us-central1-b
CLUSTER_MODE: separated
- only:
- - master
+ when: manual
+ only: ['master']
-coreos-calico:
- stage: deploy-gce-separated
+centos7-calico:
+ stage: deploy-gce-special
<<: *job
<<: *gce
variables:
<<: *gce_variables
KUBE_NETWORK_PLUGIN: calico
- CLOUD_IMAGE: coreos-stable
- CLOUD_REGION: us-west1-b
- CLUSTER_MODE: separated
- BOOTSTRAP_OS: coreos
- only:
- - master
-
+ CLOUD_IMAGE: centos-7
+ CLOUD_REGION: europe-west1-b
+ CLUSTER_MODE: ha
+ when: manual
+ only: ['master']
syntax-check:
<<: *job
diff --git a/.travis.yml.bak b/.travis.yml.bak
index 11e6bc881..7b948dcfe 100644
--- a/.travis.yml.bak
+++ b/.travis.yml.bak
@@ -24,24 +24,24 @@ env:
KUBE_NETWORK_PLUGIN=canal
CLOUD_IMAGE=debian-8-kubespray
CLOUD_REGION=asia-east1-a
- CLUSTER_MODE=default
+ CLUSTER_MODE=ha
- >-
KUBE_NETWORK_PLUGIN=calico
CLOUD_IMAGE=debian-8-kubespray
- CLOUD_REGION=asia-northeast1-b
+ CLOUD_REGION=europe-west1-c
CLUSTER_MODE=default
# Centos 7
- >-
KUBE_NETWORK_PLUGIN=flannel
CLOUD_IMAGE=centos-7
- CLOUD_REGION=europe-west1-c
+ CLOUD_REGION=asia-northeast1-c
CLUSTER_MODE=default
- >-
KUBE_NETWORK_PLUGIN=calico
CLOUD_IMAGE=centos-7
CLOUD_REGION=us-central1-b
- CLUSTER_MODE=default
+ CLUSTER_MODE=ha
# Redhat 7
- >-
@@ -55,7 +55,7 @@ env:
# KUBE_NETWORK_PLUGIN=weave
# CLOUD_IMAGE=coreos-stable
# CLOUD_REGION=europe-west1-b
- # CLUSTER_MODE=default
+ # CLUSTER_MODE=ha
# BOOTSTRAP_OS=coreos
- >-
KUBE_NETWORK_PLUGIN=canal
@@ -68,7 +68,7 @@ env:
- >-
KUBE_NETWORK_PLUGIN=canal
CLOUD_IMAGE=rhel-7
- CLOUD_REGION=europe-west1-c
+ CLOUD_REGION=asia-northeast1-b
CLUSTER_MODE=separate
- >-
KUBE_NETWORK_PLUGIN=weave
@@ -84,7 +84,7 @@ env:
matrix:
allow_failures:
- - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=default BOOTSTRAP_OS=coreos
+ - env: KUBE_NETWORK_PLUGIN=weave CLOUD_IMAGE=coreos-stable CLOUD_REGION=europe-west1-b CLUSTER_MODE=ha BOOTSTRAP_OS=coreos
before_install:
# Install Ansible.
@@ -150,6 +150,7 @@ script:
after_script:
- >
$HOME/.local/bin/ansible-playbook -i inventory/inventory.ini tests/cloud_playbooks/delete-gce.yml -c local $LOG_LEVEL
+ -e mode=${CLUSTER_MODE}
-e test_id=${TEST_ID}
-e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
-e gce_project_id=${GCE_PROJECT_ID}
diff --git a/docs/test_cases.md b/docs/test_cases.md
index 74c6d2bc8..11e4db6aa 100644
--- a/docs/test_cases.md
+++ b/docs/test_cases.md
@@ -8,19 +8,21 @@ Here is the test matrix for the Travis CI gates:
| Network plugin| OS type| GCE region| Nodes layout|
|-------------------------|-------------------------|-------------------------|-------------------------|
-| canal| debian-8-kubespray| asia-east1-a| default|
-| calico| debian-8-kubespray| asia-northeast1-b| default|
-| flannel| centos-7| europe-west1-c| default|
-| calico| centos-7| us-central1-b| default|
+| canal| debian-8-kubespray| asia-east1-a| ha|
+| calico| debian-8-kubespray| europe-west1-c| default|
+| flannel| centos-7| asia-northeast1-c| default|
+| calico| centos-7| us-central1-b| ha|
| weave| rhel-7| us-east1-c| default|
| canal| coreos-stable| us-west1-b| default|
-| canal| rhel-7| europe-west1-c| separate|
+| canal| rhel-7| asia-northeast1-b| separate|
| weave| ubuntu-1604-xenial| europe-west1-d| separate|
| calico| coreos-stable| us-central1-f| separate|
-Where the nodes layout `default` is that is given in the example inventory file.
-And the `separate` layout is when there is only node of each type, which is a kube master,
-compute and etcd cluster member.
+Where the nodes layout `default` is a non-HA two nodes setup with the separate `kube-node`
+and the `etcd` group merged with the `kube-master`. The `separate` layout is when
+there is only node of each type, which is a kube master, compute and etcd cluster member.
+And the `ha` layout stands for a two etcd nodes, two masters and a single worker node,
+partially intersecting though.
Note, the canal network plugin deploys flannel as well plus calico policy controller.
@@ -30,4 +32,23 @@ bash scripts/gen_matrix.sh
```
will (hopefully) generate the CI test cases from the current ``.travis.yml``.
+Gitlab CI test matrix
+=====================
+
+GCE instances
+-------------
+| Stage| Network plugin| OS type| GCE region| Nodes layout| When|
+|------------------|------------------|------------------|------------------|------------------|------------------|
+| part1| calico| coreos-stable| us-west1-b| separated| on_success|
+| part1| canal|debian-8-kubespray| us-east1-b| ha| manual|
+| part1| weave| rhel-7| europe-west1-b| default| manual|
+| part2| flannel| centos-7| us-west1-a| default| manual|
+| part2| calico|debian-8-kubespray| us-central1-b| default| manual|
+| part2| canal| coreos-stable| us-east1-b| default| manual|
+| special| canal| rhel-7| us-east1-b| separated| manual|
+| special| weave|ubuntu-1604-xenial| us-central1-b| separated| manual|
+| special| calico| centos-7| europe-west1-b| ha| manual|
+
+The "Stage" means a build step of the build pipeline. The steps are ordered as `part1->part2->special`.
+The "When" stands for the build step trigger condition.
diff --git a/scripts/gen_matrix.sh b/scripts/gen_matrix.sh
index 5cb04647d..c3d9f588a 100644
--- a/scripts/gen_matrix.sh
+++ b/scripts/gen_matrix.sh
@@ -1,6 +1,7 @@
#!/bin/sh
# Generates MD formatted CI matrix from the .travis.yml
-a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml)
+a=$(perl -ne '/^\s{6}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE)=(\S+)$/ && print "$2\n"' .travis.yml.bak)
+echo Travis-CI
printf "|%25s|%25s|%25s|%25s|\n" "Network plugin" "OS type" "GCE region" "Nodes layout"
echo "|-------------------------|-------------------------|-------------------------|-------------------------|"
c=0
@@ -9,3 +10,15 @@ for i in `echo $a`; do
[ $(($c % 4)) -eq 3 ] && printf "|\n"
c=$(( c + 1))
done
+
+echo
+a=$(perl -ne '/^\s{2}(stage:\sdeploy-gce-|when:\s)(\S+)$/ && print "$2\n";/^\s{4}(CLOUD_IMAGE|KUBE_NETWORK_PLUGIN|CLOUD_REGION|CLUSTER_MODE):\s(\S+)$/ && print "$2\n"' .gitlab-ci.yml)
+echo Gitlab-CI
+printf "|%18s|%18s|%18s|%18s|%18s|%18s|\n" "Stage" "Network plugin" "OS type" "GCE region" "Nodes layout" "When"
+echo "|------------------|------------------|------------------|------------------|------------------|------------------|"
+c=0
+for i in `echo $a`; do
+ printf "|%18s" $i
+ [ $(($c % 6)) -eq 5 ] && printf "|\n"
+ c=$(( c + 1))
+done
diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml
index 6084d2746..fe8bad77a 100644
--- a/tests/cloud_playbooks/create-gce.yml
+++ b/tests/cloud_playbooks/create-gce.yml
@@ -4,15 +4,24 @@
gather_facts: no
vars:
cloud_machine_type: g1-small
+ mode: default
tasks:
- name: replace_test_id
set_fact:
test_name: "{{test_id |regex_replace('\\.', '-')}}"
+ - set_fact:
+ instance_names: >-
+ {%- if mode in ['separate', 'ha'] -%}
+ k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3
+ {%- else -%}
+ k8s-{{test_name}}-1,k8s-{{test_name}}-2
+ {%- endif -%}
+
- name: Create gce instances
gce:
- instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
+ instance_names: "{{instance_names}}"
machine_type: "{{ cloud_machine_type }}"
image: "{{ cloud_image }}"
service_account_email: "{{ gce_service_account_email }}"
diff --git a/tests/cloud_playbooks/delete-gce.yml b/tests/cloud_playbooks/delete-gce.yml
index 7705c649e..be2137a3c 100644
--- a/tests/cloud_playbooks/delete-gce.yml
+++ b/tests/cloud_playbooks/delete-gce.yml
@@ -4,15 +4,24 @@
gather_facts: no
vars:
cloud_machine_type: f1-micro
+ mode: default
tasks:
- name: replace_test_id
set_fact:
test_name: "{{test_id |regex_replace('\\.', '-')}}"
+ - set_fact:
+ instance_names: >-
+ {%- if mode in ['separate', 'ha'] -%}
+ k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3
+ {%- else -%}
+ k8s-{{test_name}}-1,k8s-{{test_name}}-2
+ {%- endif -%}
+
- name: delete gce instances
gce:
- instance_names: "k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3"
+ instance_names: "{{instance_names}}"
machine_type: "{{ cloud_machine_type }}"
image: "{{ cloud_image }}"
service_account_email: "{{ gce_service_account_email }}"
diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2
index 418910771..3ede8f529 100644
--- a/tests/templates/inventory-gce.j2
+++ b/tests/templates/inventory-gce.j2
@@ -1,6 +1,8 @@
node1 ansible_ssh_host={{gce.instance_data[0].public_ip}}
node2 ansible_ssh_host={{gce.instance_data[1].public_ip}}
+{% if mode is defined and mode in ["separate", "ha"] %}
node3 ansible_ssh_host={{gce.instance_data[2].public_ip}}
+{% endif %}
{% if mode is defined and mode == "separate" %}
[kube-master]
@@ -11,19 +13,26 @@ node2
[etcd]
node3
-{% else %}
+{% elif mode is defined and mode == "ha" %}
[kube-master]
node1
node2
[kube-node]
-node1
-node2
node3
[etcd]
+node2
+node3
+{% else %}
+[kube-master]
node1
+
+[kube-node]
node2
+
+[etcd]
+node1
{% endif %}
[k8s-cluster:children]
--
GitLab