From 9c1701f2aae3e2d8f411f18d5ee98a0864777f6e Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Mon, 13 Feb 2017 14:13:28 +0300
Subject: [PATCH] Add synthetic scale deployment mode
New deploy modes: scale, ha-scale, separate-scale
Creates 200 fake hosts for deployment with fake hostvars.
Useful for testing certificate generation and propagation to other
master nodes.
Updated test cases descriptions.
---
.gitignore | 1 +
.gitlab-ci.yml | 19 +++++++++----
docs/test_cases.md | 41 +++++++++++++++++++---------
roles/etcd/tasks/check_certs.yml | 1 +
tests/cloud_playbooks/create-gce.yml | 14 +++++++++-
tests/templates/fake_hosts.yml.j2 | 3 ++
tests/templates/inventory-gce.j2 | 16 ++++++++---
7 files changed, 71 insertions(+), 24 deletions(-)
create mode 100644 tests/templates/fake_hosts.yml.j2
diff --git a/.gitignore b/.gitignore
index 074dd431d..b9b150a3d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
.vagrant
*.retry
inventory/vagrant_ansible_inventory
+inventory/group_vars/fake_hosts.yml
temp
.idea
.tox
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9c6630306..d326aecbd 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -124,6 +124,7 @@ before_script:
-e local_release_dir=${PWD}/downloads
-e resolvconf_mode=${RESOLVCONF_MODE}
-e vault_deployment_type=${VAULT_DEPLOYMENT}
+ --limit "all:!fake_hosts"
cluster.yml
# Repeat deployment if testing upgrade
@@ -150,18 +151,19 @@ before_script:
-e resolvconf_mode=${RESOLVCONF_MODE}
-e weave_cpu_requests=${WEAVE_CPU_LIMIT}
-e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+ --limit "all:!fake_hosts"
$PLAYBOOK;
fi
# Tests Cases
## Test Master API
- - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/010_check-apiserver.yml $LOG_LEVEL
+ - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $LOG_LEVEL
## Ping the between 2 pod
- - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/030_check-network.yml $LOG_LEVEL
+ - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $LOG_LEVEL
## Advanced DNS checks
- - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root tests/testcases/040_check-network-adv.yml $LOG_LEVEL
+ - ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH} -u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $LOG_LEVEL
## Idempotency checks 1/5 (repeat deployment)
- >
@@ -178,6 +180,7 @@ before_script:
-e local_release_dir=${PWD}/downloads
-e etcd_deployment_type=${ETCD_DEPLOYMENT}
-e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+ --limit "all:!fake_hosts"
cluster.yml;
fi
@@ -186,6 +189,7 @@ before_script:
if [ "${IDEMPOT_CHECK}" = "true" ]; then
ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH}
-u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
+ --limit "all:!fake_hosts"
tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
fi
@@ -197,7 +201,8 @@ before_script:
--private-key=${HOME}/.ssh/id_rsa
-e bootstrap_os=${BOOTSTRAP_OS}
-e ansible_python_interpreter=${PYPATH}
- -e reset_confirmation=yes
+ -e reset_confirmation=yes
+ --limit "all:!fake_hosts"
reset.yml;
fi
@@ -216,6 +221,7 @@ before_script:
-e local_release_dir=${PWD}/downloads
-e etcd_deployment_type=${ETCD_DEPLOYMENT}
-e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+ --limit "all:!fake_hosts"
cluster.yml;
fi
@@ -224,6 +230,7 @@ before_script:
if [ "${IDEMPOT_CHECK}" = "true" ]; then
ansible-playbook -i inventory/inventory.ini -e ansible_python_interpreter=${PYPATH}
-u $SSH_USER -e ansible_ssh_user=$SSH_USER $SSH_ARGS -b --become-user=root
+ --limit "all:!fake_hosts"
tests/testcases/040_check-network-adv.yml $LOG_LEVEL;
fi
@@ -244,7 +251,7 @@ before_script:
.coreos_calico_sep_variables: &coreos_calico_sep_variables
# stage: deploy-gce-part1
KUBE_NETWORK_PLUGIN: calico
- CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111
+ CLOUD_IMAGE: coreos-stable-1235-6-0-v20170111
CLOUD_REGION: us-west1-b
CLUSTER_MODE: separate
BOOTSTRAP_OS: coreos
@@ -310,7 +317,7 @@ before_script:
KUBE_NETWORK_PLUGIN: calico
CLOUD_IMAGE: centos-7
CLOUD_REGION: europe-west1-b
- CLUSTER_MODE: ha
+ CLUSTER_MODE: ha-scale
IDEMPOT_CHECK: "true"
.coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
diff --git a/docs/test_cases.md b/docs/test_cases.md
index 20b052861..5e232b63a 100644
--- a/docs/test_cases.md
+++ b/docs/test_cases.md
@@ -4,25 +4,40 @@ Travis CI test matrix
GCE instances
-------------
-Here is the test matrix for the Travis CI gates:
+Here is the test matrix for the CI gates:
| Network plugin| OS type| GCE region| Nodes layout|
|-------------------------|-------------------------|-------------------------|-------------------------|
-| canal| debian-8-kubespray| asia-east1-a| ha|
+| canal| debian-8-kubespray| asia-east1-a| ha-scale|
| calico| debian-8-kubespray| europe-west1-c| default|
| flannel| centos-7| asia-northeast1-c| default|
| calico| centos-7| us-central1-b| ha|
| weave| rhel-7| us-east1-c| default|
-| canal| coreos-stable| us-west1-b| default|
+| canal| coreos-stable| us-west1-b| ha-scale|
| canal| rhel-7| asia-northeast1-b| separate|
| weave| ubuntu-1604-xenial| europe-west1-d| separate|
| calico| coreos-stable| us-central1-f| separate|
-Where the nodes layout `default` is a non-HA two nodes setup with the separate `kube-node`
-and the `etcd` group merged with the `kube-master`. The `separate` layout is when
-there is only node of each type, which is a kube master, compute and etcd cluster member.
-And the `ha` layout stands for a two etcd nodes, two masters and a single worker node,
-partially intersecting though.
+
+Node Layouts
+------------
+
+There are four node layout types: `default`, `separate`, `ha`, and `scale`.
+
+
+`default` is a non-HA two nodes setup with one separate `kube-node`
+and the `etcd` group merged with the `kube-master`.
+
+`separate` layout is when there is only node of each type, which includes
+ a kube-master, kube-node, and etcd cluster member.
+
+`ha` layout consists of two etcd nodes, two masters and a single worker node,
+with role intersection.
+
+`scale` layout can be combined with above layouts. It includes 200 fake hosts
+in the Ansible inventory. This helps test TLS certificate generation at scale
+to prevent regressions and profile certain long-running tasks. These nodes are
+never actually deployed, but certificates are generated for them.
Note, the canal network plugin deploys flannel as well plus calico policy controller.
@@ -40,15 +55,15 @@ GCE instances
| Stage| Network plugin| OS type| GCE region| Nodes layout
|--------------------|--------------------|--------------------|--------------------|--------------------|
-| part1| calico| coreos-stable| us-west1-b| separated|
+| part1| calico| coreos-stable| us-west1-b| separate|
| part1| canal| debian-8-kubespray| us-east1-b| ha|
-| part1| weave| rhel-7| europe-west1-b| default|
+| part1| weave| rhel-7| europe-west1-b| scale|
| part2| flannel| centos-7| us-west1-a| default|
| part2| calico| debian-8-kubespray| us-central1-b| default|
| part2| canal| coreos-stable| us-east1-b| default|
-| special| canal| rhel-7| us-east1-b| separated|
-| special| weave| ubuntu-1604-xenial| us-central1-b| separated|
-| special| calico| centos-7| europe-west1-b| ha|
+| special| canal| rhel-7| us-east1-b| separate|
+| special| weave| ubuntu-1604-xenial| us-central1-b| scale|
+| special| calico| centos-7| europe-west1-b| ha-scale|
| special| weave| coreos-alpha| us-west1-a| ha|
The "Stage" means a build step of the build pipeline. The steps are ordered as `part1->part2->special`.
diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml
index 270c0abfc..bc14e255f 100644
--- a/roles/etcd/tasks/check_certs.yml
+++ b/roles/etcd/tasks/check_certs.yml
@@ -2,6 +2,7 @@
- name: "Check_certs | check if all certs have already been generated on first master"
stat:
path: "{{ etcd_cert_dir }}/{{ item }}"
+ get_md5: no
delegate_to: "{{groups['etcd'][0]}}"
register: etcdcert_master
run_once: true
diff --git a/tests/cloud_playbooks/create-gce.yml b/tests/cloud_playbooks/create-gce.yml
index 4ae32b787..d1e7c011e 100644
--- a/tests/cloud_playbooks/create-gce.yml
+++ b/tests/cloud_playbooks/create-gce.yml
@@ -13,7 +13,7 @@
- set_fact:
instance_names: >-
- {%- if mode in ['separate', 'ha'] -%}
+ {%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}
k8s-{{test_name}}-1,k8s-{{test_name}}-2,k8s-{{test_name}}-3
{%- else -%}
k8s-{{test_name}}-1,k8s-{{test_name}}-2
@@ -39,6 +39,18 @@
src: ../templates/inventory-gce.j2
dest: "{{ inventory_path }}"
+ - name: Make group_vars directory
+ file:
+ path: "{{ inventory_path|dirname }}/group_vars"
+ state: directory
+ when: mode in ['scale', 'separate-scale', 'ha-scale']
+
+ - name: Template fake hosts group vars
+ template:
+ src: ../templates/fake_hosts.yml.j2
+ dest: "{{ inventory_path|dirname }}/group_vars/fake_hosts.yml"
+ when: mode in ['scale', 'separate-scale', 'ha-scale']
+
- name: Wait for SSH to come up
wait_for: host={{item.public_ip}} port=22 delay=10 timeout=180 state=started
with_items: "{{gce.instance_data}}"
diff --git a/tests/templates/fake_hosts.yml.j2 b/tests/templates/fake_hosts.yml.j2
new file mode 100644
index 000000000..673109213
--- /dev/null
+++ b/tests/templates/fake_hosts.yml.j2
@@ -0,0 +1,3 @@
+ansible_default_ipv4:
+ address: 255.255.255.255
+ansible_hostname: "{{ '{{' }}inventory_hostname}}"
diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2
index f5326229c..8274fae8c 100644
--- a/tests/templates/inventory-gce.j2
+++ b/tests/templates/inventory-gce.j2
@@ -2,12 +2,11 @@
{% set node2 = gce.instance_data[1].name %}
{{node1}} ansible_ssh_host={{gce.instance_data[0].public_ip}}
{{node2}} ansible_ssh_host={{gce.instance_data[1].public_ip}}
-{% if mode is defined and mode in ["separate", "ha"] %}
+{% if mode is defined and mode in ["separate", "separate-scale"] %}
{% set node3 = gce.instance_data[2].name %}
{{node3}} ansible_ssh_host={{gce.instance_data[2].public_ip}}
{% endif %}
-
-{% if mode is defined and mode == "separate" %}
+{% if mode is defined and mode in ["separate", "separate-scale"] %}
[kube-master]
{{node1}}
@@ -19,7 +18,7 @@
[vault]
{{node3}}
-{% elif mode is defined and mode == "ha" %}
+{% elif mode is defined and mode in ["ha", "ha-scale"] %}
[kube-master]
{{node1}}
{{node2}}
@@ -51,3 +50,12 @@
[k8s-cluster:children]
kube-node
kube-master
+
+{% if mode is defined and mode in ["scale", "separate-scale", "ha-scale"] %}
+[fake_hosts]
+fake_scale_host[1:200]
+
+[kube-node:children]
+fake_hosts
+{% endif %}
+
--
GitLab