From e9c891324859899718963c7ddd6acded38996df8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Necatican=20Y=C4=B1ld=C4=B1r=C4=B1m?=
<necaticanyildirim@gmail.com>
Date: Tue, 22 Feb 2022 19:53:16 +0300
Subject: [PATCH] Add kubeadm option to etcd_deployment_type to replace the
etcd_kubeadm_enabled variable (#8317)
* Add kubeadm option to etcd_deployment_type to replace the etcd_kubeadm_enabled variable
Signed-off-by: necatican <necaticanyildirim@gmail.com>
* Add etcd kubeadm deployment documentation
Signed-off-by: necatican <necaticanyildirim@gmail.com>
* Refactor warning for the deprecated 'etcd_kubeadm_enabled' variable
Signed-off-by: necatican <necaticanyildirim@gmail.com>
---
cluster.yml | 4 +--
docs/cri-o.md | 8 +----
docs/etcd.md | 16 +++++++++
docs/upgrades.md | 12 +++++++
inventory/sample/group_vars/all/all.yml | 6 ----
inventory/sample/group_vars/all/etcd.yml | 16 +++++++++
inventory/sample/group_vars/etcd.yml | 11 ------
.../download/templates/kubeadm-images.yaml.j2 | 2 +-
roles/etcdctl/tasks/main.yml | 2 +-
.../control-plane/defaults/main/main.yml | 3 --
.../control-plane/tasks/kubeadm-etcd.yml | 2 +-
.../tasks/kubeadm-fix-apiserver.yml | 2 +-
.../control-plane/tasks/kubeadm-upgrade.yml | 4 +--
roles/kubernetes/control-plane/tasks/main.yml | 2 +-
.../templates/kubeadm-config.v1beta2.yaml.j2 | 4 +--
roles/kubernetes/kubeadm/defaults/main.yml | 3 --
roles/kubernetes/kubeadm/tasks/main.yml | 2 +-
.../preinstall/tasks/0020-verify-settings.yml | 36 ++++++++++++++-----
.../preinstall/tasks/0040-set_facts.yml | 2 +-
roles/kubespray-defaults/defaults/main.yaml | 5 +--
roles/kubespray-defaults/tasks/main.yaml | 9 +++++
scale.yml | 2 +-
tests/files/packet_ubuntu16-flannel-ha.yml | 2 +-
upgrade-cluster.yml | 4 +--
24 files changed, 99 insertions(+), 60 deletions(-)
create mode 100644 inventory/sample/group_vars/all/etcd.yml
diff --git a/cluster.yml b/cluster.yml
index 35c6fdbea..e13575e9c 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -46,7 +46,7 @@
vars:
etcd_cluster_setup: true
etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
- when: not etcd_kubeadm_enabled| default(false)
+ when: etcd_deployment_type != "kubeadm"
- hosts: k8s_cluster
gather_facts: False
@@ -59,7 +59,7 @@
vars:
etcd_cluster_setup: false
etcd_events_cluster_setup: false
- when: not etcd_kubeadm_enabled| default(false)
+ when: etcd_deployment_type != "kubeadm"
- hosts: k8s_cluster
gather_facts: False
diff --git a/docs/cri-o.md b/docs/cri-o.md
index 61a795f1d..43be723e1 100644
--- a/docs/cri-o.md
+++ b/docs/cri-o.md
@@ -13,7 +13,7 @@ _To use the CRI-O container runtime set the following variables:_
```yaml
download_container: false
skip_downloads: false
-etcd_kubeadm_enabled: true
+etcd_deployment_type: host # optionally kubeadm
```
## k8s_cluster/k8s_cluster.yml
@@ -22,12 +22,6 @@ etcd_kubeadm_enabled: true
container_manager: crio
```
-## etcd.yml
-
-```yaml
-etcd_deployment_type: host # optionally and mutually exclusive with etcd_kubeadm_enabled
-```
-
## all/crio.yml
Enable docker hub registry mirrors
diff --git a/docs/etcd.md b/docs/etcd.md
index 2d42ffb10..17aa291f5 100644
--- a/docs/etcd.md
+++ b/docs/etcd.md
@@ -1,5 +1,21 @@
# etcd
+## Deployment Types
+
+It is possible to deploy etcd with three methods. To change the default deployment method (host), use the `etcd_deployment_type` variable. Possible values are `host`, `kubeadm`, and `docker`.
+
+### Host
+
+Host deployment is the default method. Using this method will result in etcd installed as a systemd service.
+
+### Docker
+
+Installs docker in etcd group members and runs etcd on docker containers. Only usable when `container_manager` is set to `docker`.
+
+### Kubeadm
+
+This deployment method is experimental and is only available for new deployments. This deploys etcd as a static pod in master hosts.
+
## Metrics
To expose metrics on a separate HTTP port, define it in the inventory with:
diff --git a/docs/upgrades.md b/docs/upgrades.md
index 09a11d258..f494112b4 100644
--- a/docs/upgrades.md
+++ b/docs/upgrades.md
@@ -308,6 +308,18 @@ caprica Ready master,node 7h40m v1.14.1
```
+## Upgrading to v2.19
+
+`etcd_kubeadm_enabled` is being deprecated at v2.19. The same functionality is achievable by setting `etcd_deployment_type` to `kubeadm`.
+Deploying etcd using kubeadm is experimental and is only available for either new or deployments where `etcd_kubeadm_enabled` was set to `true` while deploying the cluster.
+
+From 2.19 and onward `etcd_deployment_type` variable will be placed in `group_vars/all/etcd.yml` instead of `group_vars/etcd.yml`, due to scope issues.
+The placement of the variable is only important for `etcd_deployment_type: kubeadm` right now. However, since this might change in future updates, it is recommended to move the variable.
+
+Upgrading is straightforward; no changes are required if `etcd_kubeadm_enabled` was not set to `true` when deploying.
+
+If you have a cluster where `etcd` was deployed using `kubeadm`, you will need to remove `etcd_kubeadm_enabled` the variable. Then move `etcd_deployment_type` variable from `group_vars/etcd.yml` to `group_vars/all/etcd.yml` due to scope issues and set `etcd_deployment_type` to `kubeadm`.
+
## Upgrade order
As mentioned above, components are upgraded in the order in which they were
diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml
index b4b8212e0..d3c27ac35 100644
--- a/inventory/sample/group_vars/all/all.yml
+++ b/inventory/sample/group_vars/all/all.yml
@@ -1,10 +1,4 @@
---
-## Directory where etcd data stored
-etcd_data_dir: /var/lib/etcd
-
-## Experimental kubeadm etcd deployment mode. Available only for new deployment
-etcd_kubeadm_enabled: false
-
## Directory where the binaries will be installed
bin_dir: /usr/local/bin
diff --git a/inventory/sample/group_vars/all/etcd.yml b/inventory/sample/group_vars/all/etcd.yml
new file mode 100644
index 000000000..7206a06b5
--- /dev/null
+++ b/inventory/sample/group_vars/all/etcd.yml
@@ -0,0 +1,16 @@
+---
+## Directory where etcd data stored
+etcd_data_dir: /var/lib/etcd
+
+## Container runtime
+## docker for docker, crio for cri-o and containerd for containerd.
+## Additionally you can set this to kubeadm if you want to install etcd using kubeadm
+## Kubeadm etcd deployment is experimental and only available for new deployments
+## If this is not set, container manager will be inherited from the Kubespray defaults
+## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want.
+## Also this makes possible to use different container manager for etcd nodes.
+# container_manager: containerd
+
+## Settings for etcd deployment type
+# Set this to docker if you are using container_manager: docker
+etcd_deployment_type: host
\ No newline at end of file
diff --git a/inventory/sample/group_vars/etcd.yml b/inventory/sample/group_vars/etcd.yml
index e7070a08a..cc09942fc 100644
--- a/inventory/sample/group_vars/etcd.yml
+++ b/inventory/sample/group_vars/etcd.yml
@@ -17,14 +17,3 @@
### ETCD: disable peer client cert authentication.
# This affects ETCD_PEER_CLIENT_CERT_AUTH variable
# etcd_peer_client_auth: true
-
-## Container runtime
-## docker for docker, crio for cri-o and containerd for containerd.
-## If this is not set, container manager will be inherited from the Kubespray defaults
-## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want.
-## Also this makes possible to use different container manager for etcd nodes.
-# container_manager: containerd
-
-## Settings for etcd deployment type
-# Set this to docker if you are using container_manager: docker
-etcd_deployment_type: host
diff --git a/roles/download/templates/kubeadm-images.yaml.j2 b/roles/download/templates/kubeadm-images.yaml.j2
index 7cca0e058..565533087 100644
--- a/roles/download/templates/kubeadm-images.yaml.j2
+++ b/roles/download/templates/kubeadm-images.yaml.j2
@@ -8,7 +8,7 @@ kind: ClusterConfiguration
imageRepository: {{ kube_image_repo }}
kubernetesVersion: {{ kube_version }}
etcd:
-{% if etcd_kubeadm_enabled %}
+{% if etcd_deployment_type == "kubeadm" %}
local:
imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
imageTag: "{{ etcd_image_tag }}"
diff --git a/roles/etcdctl/tasks/main.yml b/roles/etcdctl/tasks/main.yml
index a37603ade..fca078c4e 100644
--- a/roles/etcdctl/tasks/main.yml
+++ b/roles/etcdctl/tasks/main.yml
@@ -4,7 +4,7 @@
- name: Check unintentional include of this role
assert:
- that: etcd_kubeadm_enabled
+ that: etcd_deployment_type == "kubeadm"
- name: Check if etcdctl exist
stat:
diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml
index 38b3c1e3e..65ba43a46 100644
--- a/roles/kubernetes/control-plane/defaults/main/main.yml
+++ b/roles/kubernetes/control-plane/defaults/main/main.yml
@@ -2,9 +2,6 @@
# disable upgrade cluster
upgrade_cluster_setup: false
-# Experimental kubeadm etcd deployment mode. Available only for new deployment
-etcd_kubeadm_enabled: false
-
# change to 0.0.0.0 to enable insecure access from anywhere (not recommended)
kube_apiserver_insecure_bind_address: 127.0.0.1
diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml b/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml
index ebba677cc..8c0c47bb7 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-etcd.yml
@@ -15,4 +15,4 @@
- name: Ensure etcdctl script is installed
import_role:
name: etcdctl
- when: etcd_kubeadm_enabled
+ when: etcd_deployment_type == "kubeadm"
diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml b/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml
index 038a193c9..8f2f38e26 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-fix-apiserver.yml
@@ -21,4 +21,4 @@
dest: "{{ kube_config_dir }}/manifests/kube-apiserver.yaml"
regexp: '^ - --etcd-servers='
line: ' - --etcd-servers={{ etcd_access_addresses }}'
- when: not etcd_kubeadm_enabled | default(false)
+ when: etcd_deployment_type != "kubeadm"
diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
index 769ff3107..8459362cd 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml
@@ -18,7 +18,7 @@
--config={{ kube_config_dir }}/kubeadm-config.yaml
--ignore-preflight-errors=all
--allow-experimental-upgrades
- --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }}
+ --etcd-upgrade={{ etcd_deployment_type == "kubeadm" | bool | lower }}
--force
register: kubeadm_upgrade
# Retry is because upload config sometimes fails
@@ -39,7 +39,7 @@
--config={{ kube_config_dir }}/kubeadm-config.yaml
--ignore-preflight-errors=all
--allow-experimental-upgrades
- --etcd-upgrade={{ etcd_kubeadm_enabled | bool | lower }}
+ --etcd-upgrade={{ etcd_deployment_type == "kubeadm" | bool | lower }}
--force
register: kubeadm_upgrade
when: inventory_hostname != first_kube_control_plane
diff --git a/roles/kubernetes/control-plane/tasks/main.yml b/roles/kubernetes/control-plane/tasks/main.yml
index 7fa3d1b8d..bd8029af2 100644
--- a/roles/kubernetes/control-plane/tasks/main.yml
+++ b/roles/kubernetes/control-plane/tasks/main.yml
@@ -69,7 +69,7 @@
- name: Include kubeadm etcd extra tasks
include_tasks: kubeadm-etcd.yml
- when: etcd_kubeadm_enabled
+ when: etcd_deployment_type == "kubeadm"
- name: Include kubeadm secondary server apiserver fixes
include_tasks: kubeadm-fix-apiserver.yml
diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2
index ba14a9586..c329fcabe 100644
--- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2
@@ -33,7 +33,7 @@ apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
clusterName: {{ cluster_name }}
etcd:
-{% if not etcd_kubeadm_enabled %}
+{% if etcd_deployment_type != "kubeadm" %}
external:
endpoints:
{% for endpoint in etcd_access_addresses.split(',') %}
@@ -42,7 +42,7 @@ etcd:
caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
-{% elif etcd_kubeadm_enabled %}
+{% elif etcd_deployment_type == "kubeadm" %}
local:
imageRepository: "{{ etcd_image_repo | regex_replace("/etcd$","") }}"
imageTag: "{{ etcd_image_tag }}"
diff --git a/roles/kubernetes/kubeadm/defaults/main.yml b/roles/kubernetes/kubeadm/defaults/main.yml
index b6ff3fc7f..0449b8ae7 100644
--- a/roles/kubernetes/kubeadm/defaults/main.yml
+++ b/roles/kubernetes/kubeadm/defaults/main.yml
@@ -10,6 +10,3 @@ kube_override_hostname: >-
{%- else -%}
{{ inventory_hostname }}
{%- endif -%}
-
-# Experimental kubeadm etcd deployment mode. Available only for new deployment
-etcd_kubeadm_enabled: false
diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
index c8dac29b6..600e0dc7b 100644
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -153,7 +153,7 @@
- name: Extract etcd certs from control plane if using etcd kubeadm mode
include_tasks: kubeadm_etcd_node.yml
when:
- - etcd_kubeadm_enabled
+ - etcd_deployment_type == "kubeadm"
- inventory_hostname not in groups['kube_control_plane']
- kube_network_plugin in ["calico", "flannel", "canal", "cilium"] or cilium_deploy_additionally | default(false) | bool
- kube_network_plugin != "calico" or calico_datastore == "etcd"
diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
index 1d68f7e0b..cf008e3f0 100644
--- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
+++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
@@ -15,7 +15,7 @@
run_once: true
when:
- not ignore_assert_errors
- - not etcd_kubeadm_enabled
+ - etcd_deployment_type != "kubeadm"
- name: Stop if non systemd OS type
assert:
@@ -277,23 +277,41 @@
when: resolvconf_mode is defined
run_once: true
-- name: Stop if etcd deployment type is not host or docker
+- name: Stop if etcd deployment type is not host, docker or kubeadm
assert:
- that: etcd_deployment_type in ['host', 'docker']
- msg: "The etcd deployment type, 'etcd_deployment_type', must be host or docker"
+ that: etcd_deployment_type in ['host', 'docker', 'kubeadm']
+ msg: "The etcd deployment type, 'etcd_deployment_type', must be host, docker or kubeadm"
when:
- inventory_hostname in groups.get('etcd',[])
- - not etcd_kubeadm_enabled
-- name: Stop if etcd deployment type is not host when container_manager != docker
+- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker
assert:
- that: etcd_deployment_type == 'host'
- msg: "The etcd deployment type, 'etcd_deployment_type', must be host when container_manager is not docker"
+ that: etcd_deployment_type in ['host', 'kubeadm']
+ msg: "The etcd deployment type, 'etcd_deployment_type', must be host or kubeadm when container_manager is not docker"
when:
- inventory_hostname in groups.get('etcd',[])
- - not etcd_kubeadm_enabled
- container_manager != 'docker'
+# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled`
+- name: Stop if etcd deployment type is not host or kubeadm when container_manager != docker and etcd_kubeadm_enabled is not defined
+ block:
+ - name: Warn the user if they are still using `etcd_kubeadm_enabled`
+ debug:
+ msg: >
+ "WARNING! => `etcd_kubeadm_enabled` is deprecated and will be removed in a future release.
+ You can set `etcd_deployment_type` to `kubeadm` instead of setting `etcd_kubeadm_enabled` to `true`."
+ changed_when: true
+
+ - name: Stop if `etcd_kubeadm_enabled` is defined and `etcd_deployment_type` is not `kubadm` or `host`
+ assert:
+ that: etcd_deployment_type == 'kubeadm'
+ msg: >
+ It is not possible to use `etcd_kubeadm_enabled` when `etcd_deployment_type` is set to {{ etcd_deployment_type }}.
+ Unset the `etcd_kubeadm_enabled` variable and set `etcd_deployment_type` to desired deployment type (`host`, `kubeadm`, `docker`) instead."
+ when: etcd_kubeadm_enabled
+ run_once: yes
+ when: etcd_kubeadm_enabled is defined
+
- name: Stop if download_localhost is enabled but download_run_once is not
assert:
that: download_run_once
diff --git a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml
index 1cfd47777..fce7c485c 100644
--- a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml
@@ -205,7 +205,7 @@
kube_etcd_cert_file: "apiserver-etcd-client.crt"
kube_etcd_key_file: "apiserver-etcd-client.key"
when:
- - etcd_kubeadm_enabled
+ - etcd_deployment_type == "kubeadm"
- name: check /usr readonly
stat:
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index f609beba7..7cf1bd7b1 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -258,7 +258,7 @@ kubelet_shutdown_grace_period: 60s
kubelet_shutdown_grace_period_critical_pods: 20s
# Whether to deploy the container engine
-deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd_deployment_type != 'host' }}"
+deploy_container_engine: "{{ inventory_hostname in groups['k8s_cluster'] or etcd_deployment_type == 'docker' }}"
# Container for runtime
container_manager: containerd
@@ -344,9 +344,6 @@ docker_registry_mirrors: []
## Empty by default so no plugins will be installed.
docker_plugins: []
-# Experimental kubeadm etcd deployment mode. Available only for new deployment
-etcd_kubeadm_enabled: false
-
# Containerd options - thse are relevant when container_manager == 'containerd'
containerd_use_systemd_cgroup: true
diff --git a/roles/kubespray-defaults/tasks/main.yaml b/roles/kubespray-defaults/tasks/main.yaml
index fe268e953..648a4af6e 100644
--- a/roles/kubespray-defaults/tasks/main.yaml
+++ b/roles/kubespray-defaults/tasks/main.yaml
@@ -22,3 +22,12 @@
- no_proxy is not defined
tags:
- always
+
+# TODO: Clean this task up when we drop backward compatibility support for `etcd_kubeadm_enabled`
+- name: Set `etcd_deployment_type` to "kubeadm" if `etcd_kubeadm_enabled` is true
+ set_fact:
+ etcd_deployment_type: kubeadm
+ when:
+ - etcd_kubeadm_enabled is defined and etcd_kubeadm_enabled
+ tags:
+ - always
diff --git a/scale.yml b/scale.yml
index 97185d96e..de1d9b5fe 100644
--- a/scale.yml
+++ b/scale.yml
@@ -55,7 +55,7 @@
- { role: kubernetes/preinstall, tags: preinstall }
- { role: container-engine, tags: "container-engine", when: deploy_container_engine }
- { role: download, tags: download, when: "not skip_downloads" }
- - { role: etcd, tags: etcd, etcd_cluster_setup: false, when: "not etcd_kubeadm_enabled|default(false)" }
+ - { role: etcd, tags: etcd, etcd_cluster_setup: false, when: "etcd_deployment_type != 'kubeadm'" }
- name: Target only workers to get kubelet installed and checking in on any new nodes(node)
hosts: kube_node
diff --git a/tests/files/packet_ubuntu16-flannel-ha.yml b/tests/files/packet_ubuntu16-flannel-ha.yml
index 6b683343f..8df48e35a 100644
--- a/tests/files/packet_ubuntu16-flannel-ha.yml
+++ b/tests/files/packet_ubuntu16-flannel-ha.yml
@@ -5,6 +5,6 @@ mode: ha
# Kubespray settings
kube_network_plugin: flannel
-etcd_kubeadm_enabled: true
+etcd_deployment_type: kubeadm
kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085
skip_non_kubeadm_warning: true
diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml
index 8455082bd..010d64271 100644
--- a/upgrade-cluster.yml
+++ b/upgrade-cluster.yml
@@ -70,7 +70,7 @@
vars:
etcd_cluster_setup: true
etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
- when: not etcd_kubeadm_enabled | default(false)
+ when: etcd_deployment_type != "kubeadm"
- hosts: k8s_cluster
gather_facts: False
@@ -83,7 +83,7 @@
vars:
etcd_cluster_setup: false
etcd_events_cluster_setup: false
- when: not etcd_kubeadm_enabled | default(false)
+ when: etcd_deployment_type != "kubeadm"
- name: Handle upgrades to master components first to maintain backwards compat.
gather_facts: False
--
GitLab