From 0cfcd39d559f3f33fe582e5c577abae2f3b7e5d2 Mon Sep 17 00:00:00 2001
From: Rong Zhang <rongzhang@alauda.io>
Date: Wed, 21 Nov 2018 17:35:40 +0800
Subject: [PATCH] Switch to kubeadm deployment mode (#3461)

* Switch to kubeadm deployment mode

Discuss:https://github.com/kubernetes-incubator/kubespray/issues/3301

* Add non-kubeadm upgrage to kubeadm cluster
---
 .gitlab-ci.yml                                | 48 +++++++++----------
 inventory/sample/group_vars/all/all.yml       |  4 +-
 roles/kubernetes/kubeadm/tasks/main.yml       |  6 +--
 .../templates/kubeadm-client.conf.v1alpha2.j2 |  4 ++
 .../templates/kubeadm-client.conf.v1alpha3.j2 |  2 +
 .../master/tasks/kubeadm-migrate-certs.yml    |  2 +
 .../kubernetes/master/tasks/kubeadm-setup.yml |  6 ++-
 .../templates/kubeadm-config.v1alpha2.yaml.j2 |  6 +++
 .../templates/kubeadm-config.v1alpha3.yaml.j2 |  4 ++
 roles/kubespray-defaults/defaults/main.yaml   |  4 +-
 tests/files/gce_coreos-calico-aio.yml         |  2 +-
 ...ade.yml => gce_debian9-calico-upgrade.yml} |  2 +-
 tests/files/gce_ubuntu-canal-ha.yml           |  4 +-
 ...nnel-sep.yml => gce_ubuntu-flannel-ha.yml} |  6 ++-
 tests/files/gce_ubuntu18-flannel-aio.yml      |  3 +-
 15 files changed, 64 insertions(+), 39 deletions(-)
 rename tests/files/{gce_debian8-calico-upgrade.yml => gce_debian9-calico-upgrade.yml} (84%)
 rename tests/files/{gce_ubuntu-flannel-sep.yml => gce_ubuntu-flannel-ha.yml} (64%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3325fdc35..edb9af764 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -244,10 +244,6 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
-.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
-# stage: deploy-part1
-  UPGRADE_TEST: "graceful"
-
 .centos_weave_kubeadm_variables: &centos_weave_kubeadm_variables
 # stage: deploy-part1
   UPGRADE_TEST: "graceful"
@@ -256,6 +252,10 @@ before_script:
 # stage: deploy-part1
   MOVED_TO_GROUP_VARS: "true"
 
+.ubuntu_canal_ha_variables: &ubuntu_canal_ha_variables
+# stage: deploy-special
+  MOVED_TO_GROUP_VARS: "true"
+
 .ubuntu_contiv_sep_variables: &ubuntu_contiv_sep_variables
 # stage: deploy-special
   MOVED_TO_GROUP_VARS: "true"
@@ -276,7 +276,7 @@ before_script:
 # stage: deploy-part2
   MOVED_TO_GROUP_VARS: "true"
 
-.debian8_calico_variables: &debian8_calico_variables
+.debian9_calico_variables: &debian9_calico_variables
 # stage: deploy-part2
   MOVED_TO_GROUP_VARS: "true"
 
@@ -302,7 +302,7 @@ before_script:
 
 .centos7_multus_calico_variables: &centos7_multus_calico_variables
 # stage: deploy-part2
-  MOVED_TO_GROUP_VARS: "true"
+  UPGRADE_TEST: "graceful"
 
 .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables
 # stage: deploy-special
@@ -317,7 +317,7 @@ before_script:
   MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_flannel_variables: &ubuntu_flannel_variables
-# stage: deploy-special
+# stage: deploy-part2
   MOVED_TO_GROUP_VARS: "true"
 
 .ubuntu_kube_router_variables: &ubuntu_kube_router_variables
@@ -378,6 +378,17 @@ gce_centos-weave-kubeadm-sep:
   except: ['triggers']
   only: [/^pr-.*$/]
 
+gce_ubuntu-flannel-ha:
+  stage: deploy-part2
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_flannel_variables
+  when: on_success
+  except: ['triggers']
+  only: [/^pr-.*$/]
+
 ### MANUAL JOBS
 
 gce_ubuntu-weave-sep:
@@ -402,7 +413,7 @@ gce_coreos-calico-sep-triggers:
   only: ['triggers']
 
 gce_ubuntu-canal-ha-triggers:
-  stage: deploy-part2
+  stage: deploy-special
   <<: *job
   <<: *gce
   variables:
@@ -444,7 +455,7 @@ do_ubuntu-canal-ha:
   only: ['master', /^pr-.*$/]
 
 gce_ubuntu-canal-ha:
-  stage: deploy-part2
+  stage: deploy-special
   <<: *job
   <<: *gce
   variables:
@@ -539,24 +550,24 @@ gce_rhel7-weave-triggers:
   when: on_success
   only: ['triggers']
 
-gce_debian8-calico-upgrade:
+gce_debian9-calico-upgrade:
   stage: deploy-part2
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *debian8_calico_variables
+    <<: *debian9_calico_variables
   when: manual
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-gce_debian8-calico-triggers:
+gce_debian9-calico-triggers:
   stage: deploy-part2
   <<: *job
   <<: *gce
   variables:
     <<: *gce_variables
-    <<: *debian8_calico_variables
+    <<: *debian9_calico_variables
   when: on_success
   only: ['triggers']
 
@@ -690,17 +701,6 @@ gce_ubuntu-rkt-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-gce_ubuntu-flannel-sep:
-  stage: deploy-special
-  <<: *job
-  <<: *gce
-  variables:
-    <<: *gce_variables
-    <<: *ubuntu_flannel_variables
-  when: manual
-  except: ['triggers']
-  only: ['master', /^pr-.*$/]
-
 gce_ubuntu-kube-router-sep:
   stage: deploy-special
   <<: *job
diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml
index 368b94448..49bf3077c 100644
--- a/inventory/sample/group_vars/all/all.yml
+++ b/inventory/sample/group_vars/all/all.yml
@@ -45,8 +45,8 @@ bin_dir: /usr/local/bin
 #cloud_provider:
 
 
-## Uncomment to enable experimental kubeadm deployment mode
-#kubeadm_enabled: false
+## kubeadm deployment mode
+kubeadm_enabled: true
 
 ## Set these proxy values in order to update package manager and docker daemon to use proxies
 #http_proxy: ""
diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
index 84fd31f69..1a2470d46 100644
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -87,9 +87,9 @@
 
 - name: Update server field in kube-proxy kubeconfig
   shell: >-
-    {{ bin_dir }}/kubectl get configmap kube-proxy -n kube-system -o yaml
+    {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf get configmap kube-proxy -n kube-system -o yaml
     | sed 's#server:.*#server:\ {{ kube_apiserver_endpoint }}#g'
-    | {{ bin_dir }}/kubectl replace -f -
+    | {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf replace -f -
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:
@@ -110,7 +110,7 @@
   when: kube_network_plugin in ['calico','canal']
 
 - name: Restart all kube-proxy pods to ensure that they load the new configmap
-  shell: "{{ bin_dir }}/kubectl delete pod -n kube-system -l k8s-app=kube-proxy"
+  shell: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf delete pod -n kube-system -l k8s-app=kube-proxy"
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
index 62105fbde..eebcdf7c0 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
@@ -18,4 +18,8 @@ nodeRegistration:
   name: {{ inventory_hostname  }}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
 {% endif %}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
index 61707ea85..a1e0887e0 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
@@ -18,6 +18,8 @@ nodeRegistration:
   name: {{ inventory_hostname  }}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
 {% else %}
   criSocket: /var/run/dockershim.sock
 {% endif %}
diff --git a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml
index 83bfbb22a..3a3a45a8e 100644
--- a/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml
@@ -15,4 +15,6 @@
     - {src: front-proxy-client-key.pem, dest: front-proxy-client.key}
     - {src: service-account-key.pem, dest: sa.pub}
     - {src: service-account-key.pem, dest: sa.key}
+    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt }
+    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key }
   register: kubeadm_copy_old_certs
diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index 21a03c963..e0c13fefa 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -6,6 +6,10 @@
   delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
 
+- name: kubeadm | Migrate old certs if necessary
+  import_tasks: kubeadm-migrate-certs.yml
+  when: old_apiserver_cert.stat.exists
+
 - name: kubeadm | Check service account key
   stat:
     path: "{{ kube_cert_dir }}/sa.key"
@@ -219,7 +223,7 @@
   when: old_apiserver_cert.stat.exists
 
 - name: kubeadm | Remove taint for master with node role
-  command: "{{ bin_dir }}/kubectl taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
+  command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
   delegate_to: "{{groups['kube-master']|first}}"
   when: inventory_hostname in groups['kube-node']
   failed_when: false
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
index c418d5aec..5f1bcab2b 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
@@ -171,11 +171,17 @@ nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
 {% endif %}
 {% if dynamic_kubelet_configuration %}
 featureGates:
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
index 84d59c93e..4adbe2b8b 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
@@ -7,11 +7,15 @@ nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
 {% else %}
   criSocket: /var/run/dockershim.sock
 {% endif %}
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 307ad2a2c..c9c8c5575 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -231,8 +231,8 @@ cert_management: script
 
 helm_deployment_type: host
 
-# Enable kubeadm deployment (experimental)
-kubeadm_enabled: false
+# Enable kubeadm deployment
+kubeadm_enabled: true
 
 # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts
 kubeconfig_localhost: false
diff --git a/tests/files/gce_coreos-calico-aio.yml b/tests/files/gce_coreos-calico-aio.yml
index 7430f5620..51a7c686d 100644
--- a/tests/files/gce_coreos-calico-aio.yml
+++ b/tests/files/gce_coreos-calico-aio.yml
@@ -1,7 +1,7 @@
 # Instance settings
 cloud_image_family: coreos-stable
 cloud_region: us-central1-a
-cloud_machine_type: "n1-standard-1"
+cloud_machine_type: "n1-standard-2"
 mode: aio
 ##user-data to simply turn off coreos upgrades
 startup_script: 'systemctl disable locksmithd && systemctl stop locksmithd'
diff --git a/tests/files/gce_debian8-calico-upgrade.yml b/tests/files/gce_debian9-calico-upgrade.yml
similarity index 84%
rename from tests/files/gce_debian8-calico-upgrade.yml
rename to tests/files/gce_debian9-calico-upgrade.yml
index 1230bfffd..b129904ff 100644
--- a/tests/files/gce_debian8-calico-upgrade.yml
+++ b/tests/files/gce_debian9-calico-upgrade.yml
@@ -1,5 +1,5 @@
 # Instance settings
-cloud_image: debian-8-kubespray
+cloud_image: debian-9-kubespray
 cloud_region: us-central1-b
 mode: default
 
diff --git a/tests/files/gce_ubuntu-canal-ha.yml b/tests/files/gce_ubuntu-canal-ha.yml
index 63f4179fb..419c0426f 100644
--- a/tests/files/gce_ubuntu-canal-ha.yml
+++ b/tests/files/gce_ubuntu-canal-ha.yml
@@ -1,7 +1,7 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-central1-c
-mode: ha
+cloud_region: us-central1-b
+mode: separate
 
 # Deployment settings
 kube_network_plugin: canal
diff --git a/tests/files/gce_ubuntu-flannel-sep.yml b/tests/files/gce_ubuntu-flannel-ha.yml
similarity index 64%
rename from tests/files/gce_ubuntu-flannel-sep.yml
rename to tests/files/gce_ubuntu-flannel-ha.yml
index e34137852..03076e26a 100644
--- a/tests/files/gce_ubuntu-flannel-sep.yml
+++ b/tests/files/gce_ubuntu-flannel-ha.yml
@@ -1,10 +1,12 @@
 # Instance settings
 cloud_image_family: ubuntu-1604-lts
-cloud_region: us-central1-a
-mode: separate
+cloud_region: us-central1-b
+cloud_machine_type: "n1-standard-1"
+mode: ha
 
 # Deployment settings
 kube_network_plugin: flannel
+kubeadm_enabled: false
 deploy_netchecker: true
 dns_min_replicas: 1
 cloud_provider: gce
diff --git a/tests/files/gce_ubuntu18-flannel-aio.yml b/tests/files/gce_ubuntu18-flannel-aio.yml
index cd3a95534..c6638b6d6 100644
--- a/tests/files/gce_ubuntu18-flannel-aio.yml
+++ b/tests/files/gce_ubuntu18-flannel-aio.yml
@@ -1,10 +1,11 @@
 # Instance settings
 cloud_image_family: ubuntu-1804-lts
 cloud_region: us-central1-a
-cloud_machine_type: "n1-standard-1"
+cloud_machine_type: "n1-standard-2"
 mode: aio
 
 # Deployment settings
+kubeadm_enabled: true
 kube_network_plugin: flannel
 dynamic_kubelet_configuration: true
 deploy_netchecker: true
-- 
GitLab