Use K8s 1.14 and add kubeadm experimental control plane mode (#4514)

* Use K8s 1.14 and add kubeadm experimental control plane mode

This reverts commit d39c273d.

* Cleanup kubeadm setup run on first master

* pin kubeadm_certificate_key in test

* Remove kubelet autolabel of kube-node, add symlink for pki dir

Change-Id: Id5e74dd667c60675dbfe4193b0bc9fb44380e1ca

README.md

@@ -108,7 +108,7 @@ Supported Components
 --------------------
 
 -   Core
-    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.13.5
+    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.14.0
     -   [etcd](https://github.com/coreos/etcd) v3.2.26
     -   [docker](https://www.docker.com/) v18.06 (see note)
     -   [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -20,7 +20,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.13.5
+kube_version: v1.14.0
 
 # kubernetes image repo define
 kube_image_repo: "gcr.io/google-containers"
@@ -153,6 +153,10 @@ etcd_deployment_type: docker
 kubelet_deployment_type: host
 helm_deployment_type: host
 
+# Enable kubeadm experimental control plane
+kubeadm_control_plane: false
+kubeadm_certificate_key: "{{ lookup('password', credentials_dir + '/kubeadm_certificate_key.creds length=64 chars=hexdigits') | lower }}"
+
 # K8s image pull policy (imagePullPolicy)
 k8s_image_pull_policy: IfNotPresent
 

roles/download/defaults/main.yml

@@ -35,7 +35,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube
 image_arch: "{{host_architecture | default('amd64')}}"
 
 # Versions
-kube_version: v1.13.5
+kube_version: v1.14.0
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.26
 

roles/kubernetes/client/tasks/main.yml

@@ -40,18 +40,35 @@
   run_once: yes
   when: kubeconfig_localhost|default(false)
 
+# NOTE(mattymo): Please forgive this workaround
 - name: Generate admin kubeconfig with external api endpoint
   shell: >-
-    {{ bin_dir }}/kubeadm alpha
-    {% if kubeadm_version is version('v1.13.0', '<') %}
-    phase
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    mkdir -p {{ kube_config_dir }}/external_kubeconfig &&
     {% endif %}
+    {{ bin_dir }}/kubeadm
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    init phase
+    {% elif kubeadm_version is version('v1.13.0', '>=') %}
+    alpha
+    {% else %}
+    alpha phase
+    {% endif %}
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    kubeconfig admin
+    --kubeconfig-dir {{ kube_config_dir }}/external_kubeconfig
+    {% else %}
     kubeconfig user
     --client-name kubernetes-admin
     --org system:masters
+    {% endif %}
     --cert-dir {{ kube_config_dir }}/ssl
     --apiserver-advertise-address {{ external_apiserver_address }}
     --apiserver-bind-port {{ external_apiserver_port }}
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    && cat {{ kube_config_dir }}/external_kubeconfig/admin.conf &&
+    rm -rf {{ kube_config_dir }}/external_kubeconfig
+    {% endif %}
   environment: "{{ proxy_env }}"
   run_once: yes
   register: admin_kubeconfig

roles/kubernetes/kubeadm/tasks/main.yml

@@ -23,9 +23,15 @@
 
 - name: Create kubeadm token for joining nodes with 24h expiration (default)
   command: "{{ bin_dir }}/kubeadm token create"
-  run_once: true
   register: temp_token
   delegate_to: "{{ groups['kube-master'][0] }}"
+  when: kubeadm_token is not defined
+
+- name: Set kubeadm_token to generated token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: kubeadm_token is not defined
+
 
 - name: gets the kubeadm version
   command: "{{ bin_dir }}/kubeadm version -o short"
@@ -61,8 +67,6 @@
     dest: "{{ kube_config_dir }}/kubeadm-client.conf"
     backup: yes
   when: not is_kube_master
-  vars:
-    kubeadm_token: "{{ temp_token.stdout }}"
 
 - name: Join to cluster if needed
   environment:
@@ -122,11 +126,10 @@
     {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf get configmap kube-proxy -n kube-system -o yaml
     | sed 's#server:.*#server:\ {{ kube_apiserver_endpoint }}#g'
     | {{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf replace -f -
-  delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:
+    - inventory_hostname == groups['kube-master']|first
     - kubeadm_config_api_fqdn is not defined
-    - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
     - not kube_proxy_remove
   tags:
@@ -134,11 +137,10 @@
 
 - name: Restart all kube-proxy pods to ensure that they load the new configmap
   shell: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf delete pod -n kube-system -l k8s-app=kube-proxy --force --grace-period=0"
-  delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:
+    - inventory_hostname == groups['kube-master']|first
     - kubeadm_config_api_fqdn is not defined
-    - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
     - not kube_proxy_remove
   tags:
@@ -159,11 +161,10 @@
 # is fixed
 - name: Delete kube-proxy daemonset if kube_proxy_remove set, e.g. kube_network_plugin providing proxy services
   shell: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf delete daemonset -n kube-system kube-proxy"
-  delegate_to: "{{groups['kube-master']|first}}"
   run_once: true
   when:
+    - inventory_hostname == groups['kube-master']|first
     - kube_proxy_remove
-    - is_kube_master
     - kubeadm_discovery_address != kube_apiserver_endpoint
   tags:
     - kube-proxy

roles/kubernetes/master/defaults/main/kube-proxy.yml

@@ -107,4 +107,4 @@ kube_proxy_resource_container: /kube-proxy
 
 # udpIdleTimeout is how long an idle UDP connection will be kept open (e.g. '250ms', '2s').
 # Must be greater than 0. Only applicable for proxyMode=userspace.
-kube_proxy_udp_idle_timeout: 250ms
+kube_proxy_udp_idle_timeout: 250ms
\ No newline at end of file

roles/kubernetes/master/defaults/main/main.yml

@@ -23,11 +23,18 @@ kube_apiserver_storage_backend: etcd3
 # By default, force back to etcd2. Set to true to force etcd3 (experimental!)
 force_etcd3: false
 
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+
 # Associated interfaces must be reachable by the rest of the cluster, and by
 # CLI/web clients.
 kube_controller_manager_bind_address: 0.0.0.0
 kube_scheduler_bind_address: 0.0.0.0
 
+# discovery_timeout modifies the discovery timeout
+discovery_timeout: 5m0s
+
 # audit support
 kubernetes_audit: false
 # path to audit log file
@@ -78,7 +85,6 @@ kube_apiserver_request_timeout: "1m0s"
 
 # 1.9 and below Admission control plug-ins
 kube_apiserver_admission_control:
-  - Initializers
   - NamespaceLifecycle
   - LimitRanger
   - ServiceAccount
@@ -99,8 +105,7 @@ kube_apiserver_enable_admission_plugins: []
 kube_apiserver_disable_admission_plugins: []
 
 # extra runtime config
-kube_api_runtime_config:
-  - admissionregistration.k8s.io/v1alpha1
+kube_api_runtime_config: []
 
 ## Enable/Disable Kube API Server Authentication Methods
 kube_basic_auth: false

roles/kubernetes/master/tasks/kubeadm-certificate.yml

@@ -12,33 +12,3 @@
     - {src: front-proxy-client.crt, dest: front-proxy-client.crt.old}
     - {src: front-proxy-client.key, dest: front-proxy-client.key.old}
   ignore_errors: yes
-
-- name: Remove old certs and keys
-  file:
-    path: "{{ kube_cert_dir }}/{{ item }}"
-    state: absent
-  with_items:
-    - apiserver.crt
-    - apiserver.key
-    - apiserver-kubelet-client.crt
-    - apiserver-kubelet-client.key
-    - front-proxy-client.crt
-    - front-proxy-client.key
-
-- name: Generate new certs and keys
-  command: "{{ bin_dir }}/kubeadm init phase certs {{ item }} --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  with_items:
-    - apiserver
-    - apiserver-kubelet-client
-    - front-proxy-client
-  when: inventory_hostname == groups['kube-master']|first and kubeadm_version is version('v1.13.0', '>=')
-
-- name: Generate new certs and keys
-  command: "{{ bin_dir }}/kubeadm alpha phase certs {{ item }} --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  with_items:
-    - apiserver
-    - apiserver-kubelet-client
-    - front-proxy-client
-  when: inventory_hostname == groups['kube-master']|first and kubeadm_version is version('v1.13.0', '<')

roles/kubernetes/master/tasks/kubeadm-kubeconfig.yml

----
-- name: Backup old configuration files
-  copy:
-    src: "{{ kube_config_dir }}/{{ item.src }}"
-    dest: "{{ kube_config_dir }}/{{ item.dest }}"
-    remote_src: yes
-  with_items:
-    - {src: admin.conf, dest: admin.conf.old}
-    - {src: kubelet.conf, dest: kubelet.conf.old}
-    - {src: controller-manager.conf, dest: controller-manager.conf.old}
-    - {src: scheduler.conf, dest: scheduler.conf.old}
-  ignore_errors: yes
-
-- name: Remove old configuration files
-  file:
-    path: "{{ kube_config_dir }}/{{ item }}"
-    state: absent
-  with_items:
-    - admin.conf
-    - kubelet.conf
-    - controller-manager.conf
-    - scheduler.conf
-
-- name: Generate new configuration files
-  command: "{{ bin_dir }}/kubeadm init phase kubeconfig all --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  when: kubeadm_version is version('v1.13.0', '>=')
-  ignore_errors: yes
-
-- name: Generate new configuration files
-  command: "{{ bin_dir }}/kubeadm alpha phase kubeconfig all --config={{ kube_config_dir }}/kubeadm-config.yaml"
-  environment: "{{ proxy_env }}"
-  when: kubeadm_version is version('v1.13.0', '<')
-  ignore_errors: yes

roles/kubernetes/master/tasks/kubeadm-migrate-certs.yml

@@ -15,6 +15,6 @@
     - {src: front-proxy-client-key.pem, dest: front-proxy-client.key}
     - {src: service-account-key.pem, dest: sa.pub}
     - {src: service-account-key.pem, dest: sa.key}
-    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt }
-    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key }
+    - {src: "node-{{ inventory_hostname }}.pem", dest: apiserver-kubelet-client.crt}
+    - {src: "node-{{ inventory_hostname }}-key.pem", dest: apiserver-kubelet-client.key}
   register: kubeadm_copy_old_certs

roles/kubernetes/master/tasks/kubeadm-secondary-experimental.yml

+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" in kube_apiserver_endpoint or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint }}
+      {%- endif %}
+  tags:
+    - facts
+
+- name: Create kubeadm ControlPlane config
+  template:
+    src: "kubeadm-controlplane.{{ kubeadmConfig_api_version }}.yaml.j2"
+    dest: "{{ kube_config_dir }}/kubeadm-controlplane.yaml"
+    backup: yes
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+
+- name: Wait for k8s apiserver
+  wait_for:
+    host: "{{kubeadm_discovery_address.split(':')[0]}}"
+    port: "{{kubeadm_discovery_address.split(':')[1]}}"
+    timeout: 180
+
+- name: Joining control plane node to the cluster.
+  command: >-
+    {{ bin_dir }}/kubeadm join
+    --config {{ kube_config_dir}}/kubeadm-controlplane.yaml
+    --ignore-preflight-errors=all
+    {% if kubeadm_certificate_key is defined %}
+    --certificate-key={{ kubeadm_certificate_key }}
+    {% endif %}
+  register: kubeadm_join_control_plane
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+
+- name: Set secret_changed to false to avoid extra token rotation
+  set_fact:
+    secret_changed: false

roles/kubernetes/master/tasks/kubeadm-secondary-legacy.yml

+---
+- name: slurp kubeadm certs
+  slurp:
+    src: "{{ item }}"
+  with_items:
+    - "{{ kube_cert_dir }}/apiserver.crt"
+    - "{{ kube_cert_dir }}/apiserver.key"
+    - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
+    - "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
+    - "{{ kube_cert_dir }}/ca.crt"
+    - "{{ kube_cert_dir }}/ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
+    - "{{ kube_cert_dir }}/front-proxy-ca.key"
+    - "{{ kube_cert_dir }}/front-proxy-client.crt"
+    - "{{ kube_cert_dir }}/front-proxy-client.key"
+    - "{{ kube_cert_dir }}/sa.key"
+    - "{{ kube_cert_dir }}/sa.pub"
+  register: kubeadm_certs
+  delegate_to: "{{ groups['kube-master']|first }}"
+
+- name: kubeadm | write out kubeadm certs
+  copy:
+    dest: "{{ item.item }}"
+    content: "{{ item.content | b64decode }}"
+    owner: root
+    group: root
+    mode: 0600
+  no_log: true
+  register: copy_kubeadm_certs
+  with_items: "{{ kubeadm_certs.results }}"
+  when: inventory_hostname != groups['kube-master']|first
+
+- name: kubeadm | Init other uninitialized masters
+  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  register: kubeadm_init
+  retries: 10
+  until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
+  when:
+    - inventory_hostname != groups['kube-master']|first
+    - not kubeadm_already_run.stat.exists
+  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
+  environment:
+    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
+  notify: Master | restart kubelet

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -10,11 +10,10 @@
   import_tasks: kubeadm-migrate-certs.yml
   when: old_apiserver_cert.stat.exists
 
-- name: kubeadm | Check apiserver key
+- name: kubeadm | Check serviceaccount key
   stat:
-    path: "{{ kube_cert_dir }}/apiserver.key"
-  register: apiserver_key_before
-  delegate_to: "{{groups['kube-master']|first}}"
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_before
   run_once: true
 
 - name: kubeadm | Check if kubeadm has already run
@@ -62,10 +61,6 @@
     sans_address: "{{ groups['kube-master'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']) | list | select('defined') | list }}"
   tags: facts
 
-- name: kubeadm | Copy etcd cert dir under k8s cert dir
-  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
-  changed_when: false
-
 - name: Create audit-policy directory
   file:
     path: "{{ audit_policy_file | dirname }}"
@@ -94,7 +89,18 @@
     - kubeadm_already_run.stat.exists
 
 - name: kubeadm | Initialize first master
-  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
+  command: >-
+    timeout -k 600s 600s
+    {{ bin_dir }}/kubeadm init
+    --config={{ kube_config_dir }}/kubeadm-config.yaml
+    --ignore-preflight-errors=all
+    {% if kubeadm_version is version('v1.14.0', '>=') %}
+    --experimental-upload-certs
+    {% endif %}
+    --skip-phases=addon/coredns
+    {% if kubeadm_certificate_key is defined %}
+    --certificate-key={{ kubeadm_certificate_key }}
+    {% endif %}
   register: kubeadm_init
   # Retry is because upload config sometimes fails
   retries: 3
@@ -105,76 +111,73 @@
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
   notify: Master | restart kubelet
 
-- name: slurp kubeadm certs
-  slurp:
-    src: "{{ item }}"
-  with_items:
-    - "{{ kube_cert_dir }}/apiserver.crt"
-    - "{{ kube_cert_dir }}/apiserver.key"
-    - "{{ kube_cert_dir }}/apiserver-kubelet-client.crt"
-    - "{{ kube_cert_dir }}/apiserver-kubelet-client.key"
-    - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/ca.key"
-    - "{{ kube_cert_dir }}/front-proxy-ca.crt"
-    - "{{ kube_cert_dir }}/front-proxy-ca.key"
-    - "{{ kube_cert_dir }}/front-proxy-client.crt"
-    - "{{ kube_cert_dir }}/front-proxy-client.key"
-    - "{{ kube_cert_dir }}/sa.key"
-    - "{{ kube_cert_dir }}/sa.pub"
-  register: kubeadm_certs
-  delegate_to: "{{ groups['kube-master']|first }}"
-  run_once: true
+- name: set kubeadm certificate key
+  set_fact:
+    kubeadm_certificate_key: "{{ item | regex_search('--certificate-key ([^ ]+)','\\1') | first }}"
+  with_items: "{{ (hostvars['kube-master'][0]['kubeadm_init']|default({'stdout_lines': []}))['stdout_lines'] }}"
+  when:
+    - kubeadm_version is version('v1.14.0', '>=')
+    - kubeadm_certificate_key is not defined
+    - item | trim | match('.*--certificate-key .*')
+    - hostvars['kube-master'][0]['kubeadm_init']['stdout_lines'] is defined
+
+- name: Create kubeadm token for joining nodes with 24h expiration (default)
+  command: "{{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create"
+  register: temp_token
+  retries: 5
+  delay: 5
+  until: temp_token is succeeded
+  delegate_to: "{{groups['kube-master']|first}}"
+  when: kubeadm_token is not defined
+  tags:
+    - kubeadm_token
 
-- name: kubeadm | write out kubeadm certs
-  copy:
-    dest: "{{ item.item }}"
-    content: "{{ item.content | b64decode }}"
-    owner: root
-    group: root
-    mode: 0600
-  no_log: true
-  register: copy_kubeadm_certs
-  with_items: "{{ kubeadm_certs.results }}"
-  when: inventory_hostname != groups['kube-master']|first
-
-- name: kubeadm | Kubeconfig management with kubeadm
-  import_tasks: kubeadm-kubeconfig.yml
+- name: Set kubeadm_token
+  set_fact:
+    kubeadm_token: "{{ temp_token.stdout }}"
+  when: temp_token.stdout is defined
+  tags:
+    - kubeadm_token
+
+- name: Create hardcoded kubeadm token for joining nodes with 24h expiration (if defined)
+  shell: >-
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token delete {{ kubeadm_token }} || :;
+    {{ bin_dir }}/kubeadm --kubeconfig /etc/kubernetes/admin.conf token create {{ kubeadm_token }}
   when:
-    - not upgrade_cluster_setup
-    - kubeadm_already_run.stat.exists
+    - inventory_hostname == groups['kube-master']|first
+    - kubeadm_token is defined
+  tags:
+    - kubeadm_token
 
-- name: kubeadm | Init other uninitialized masters
-  command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --ignore-preflight-errors=all
-  register: kubeadm_init
-  retries: 10
-  until: kubeadm_init is succeeded or "field is immutable" in kubeadm_init.stderr
-  when: inventory_hostname != groups['kube-master']|first and not kubeadm_already_run.stat.exists
-  failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr
-  environment:
-    PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
-  notify: Master | restart kubelet
+- name: kubeadm | Initialize other masters (experimental control plane)
+  include: kubeadm-secondary-experimental.yml
+  when: kubeadm_control_plane
 
-- name: kubeadm | upgrage kubernetes cluster
+- name: kubeadm | Initialize other masters (experimental control plane)
+  include: kubeadm-secondary-legacy.yml
+  when: not kubeadm_control_plane
+
+- name: kubeadm | upgrade kubernetes cluster
   import_tasks: kubeadm-upgrade.yml
   when: upgrade_cluster_setup
 
-- name: kubeadm | Check apiserver key again
+- name: kubeadm | Check serviceaccount key again
   stat:
-    path: "{{ kube_cert_dir }}/apiserver.key"
-  register: apiserver_key_after
-  delegate_to: "{{groups['kube-master']|first}}"
+    path: "{{ kube_cert_dir }}/sa.key"
+  register: sa_key_after
   run_once: true
 
 - name: kubeadm | Set secret_changed if service account key was updated
   command: /bin/true
   notify: Master | set secret_changed
-  when: apiserver_key_before.stat.checksum|default("") != apiserver_key_after.stat.checksum
+  when: sa_key_before.stat.checksum|default("") != sa_key_after.stat.checksum
 
 - name: kubeadm | cleanup old certs if necessary
   import_tasks: kubeadm-cleanup-old-certs.yml
   when:
     - old_apiserver_cert.stat.exists
 
+# FIXME(mattymo): from docs: If you don't want to taint your control-plane node, set this field to an empty slice, i.e. `taints: {}` in the YAML file.
 - name: kubeadm | Remove taint for master with node role
   command: "{{ bin_dir }}/kubectl --kubeconfig {{ kube_config_dir }}/admin.conf taint node {{ inventory_hostname }} node-role.kubernetes.io/master:NoSchedule-"
   delegate_to: "{{groups['kube-master']|first}}"

roles/kubernetes/master/tasks/kubeadm-upgrade.yml

@@ -17,6 +17,8 @@
   failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
   notify: Master | restart kubelet
 
+# FIXME: https://github.com/kubernetes/kubeadm/issues/1498 remove stdout_lines
+# check after issue is fixed
 - name: kubeadm | Upgrade other masters
   command: >-
     timeout -k 600s 600s
@@ -29,5 +31,8 @@
     --etcd-upgrade=false
   register: kubeadm_upgrade
   when: inventory_hostname != groups['kube-master']|first
-  failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
+  failed_when:
+    - kubeadm_upgrade.rc != 0
+    - '"field is immutable" not in kubeadm_upgrade.stderr'
+    - kubeadm_upgrade.stdout_lines | length > 1
   notify: Master | restart kubelet

roles/kubernetes/master/tasks/kubeadm-version.yml

@@ -3,11 +3,6 @@
   command: "{{ bin_dir }}/kubeadm version -o short"
   register: kubeadm_output
 
-- name: sets kubeadm api version to v1alpha1
-  set_fact:
-    kubeadmConfig_api_version: v1alpha1
-  when: kubeadm_output.stdout is version('v1.11.0', '<')
-
 - name: sets kubeadm api version to v1alpha2
   set_fact:
     kubeadmConfig_api_version: v1alpha2

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -3,8 +3,8 @@
   command: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} ls /registry/minions"
   environment:
     ETCDCTL_API: 2
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}"
+    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}"
   register: old_data_exists
   delegate_to: "{{groups['etcd'][0]}}"
   changed_when: false

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

-apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-api:
-{% if kubeadm_config_api_fqdn is defined %}
-  controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}
-  bindPort: {{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
-{% else %}
-  advertiseAddress: {{ ip | default(fallback_ips[inventory_hostname]) }}
-  bindPort: {{ kube_apiserver_port }}
-{% endif %}
-etcd:
-  endpoints:
-{% for endpoint in etcd_access_addresses.split(',') %}
-  - {{ endpoint }}
-{% endfor %}
-  caFile: {{ etcd_cert_dir }}/ca.pem
-  certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-  keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
-networking:
-  dnsDomain: {{ dns_domain }}
-  serviceSubnet: {{ kube_service_addresses }}
-  podSubnet: {{ kube_pods_subnet }}
-kubernetesVersion: {{ kube_version }}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
-cloudProvider: {{cloud_provider}}
-cloudConfig: {{ kube_config_dir }}/cloud_config
-{% elif cloud_provider is defined and cloud_provider in ["external"] %}
-cloudConfig: {{ kube_config_dir }}/cloud_config
-{% endif %}
-{% if kube_proxy_mode == 'ipvs' %}
-kubeProxy:
-  config:
-{% if kube_version is version('v1.10', '<') %}
-    featureGates: SupportIPVSProxyMode=true
-{% endif %}
-{% if kube_version is version('v1.10', '>=') %}
-    featureGates:
-      SupportIPVSProxyMode: true
-{% endif %}
-    mode: ipvs
-{% endif %}
-{% if kube_proxy_nodeport_addresses %}
-    nodePortAddresses: {{ kube_proxy_nodeport_addresses }}
-{% endif %}
-resourceContainer: ""
-authorizationModes:
-{% for mode in authorization_modes %}
-- {{ mode }}
-{% endfor %}
-selfHosted: false
-apiServerExtraArgs:
-  bind-address: {{ kube_apiserver_bind_address }}
-{% if kube_apiserver_insecure_port|string != "0" %}
-  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
-{% endif %}
-  insecure-port: "{{ kube_apiserver_insecure_port }}"
-{% if kube_version is version('v1.10', '<') %}
-  admission-control: {{ kube_apiserver_admission_control | join(',') }}
-{% else %}
-{% if kube_apiserver_enable_admission_plugins|length > 0 %}
-  enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
-{% endif %}
-{% if kube_apiserver_disable_admission_plugins|length > 0 %}
-  disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
-{% endif %}
-{% endif %}
-  apiserver-count: "{{ kube_apiserver_count }}"
-{% if kube_version is version('v1.9', '>=') %}
-  endpoint-reconciler-type: lease
-{% endif %}
-{% if etcd_events_cluster_enabled %}
-  etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
-{% endif %}
-  service-node-port-range: {{ kube_apiserver_node_port_range }}
-  kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
-  profiling: "{{ kube_profiling }}"
-  request-timeout: "{{ kube_apiserver_request_timeout }}"
-  repair-malformed-updates: "false"
-  enable-aggregator-routing: "{{ kube_api_aggregator_routing }}"
-{% if kube_api_anonymous_auth is defined and kube_version is version('v1.5', '>=')  %}
-  anonymous-auth: "{{ kube_api_anonymous_auth }}"
-{% endif %}
-{% if kube_basic_auth|default(true) %}
-  basic-auth-file: {{ kube_users_dir }}/known_users.csv
-{% endif %}
-{% if kube_token_auth|default(true) %}
-  token-auth-file: {{ kube_token_dir }}/known_tokens.csv
-{% endif %}
-{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
-  oidc-issuer-url: {{ kube_oidc_url }}
-  oidc-client-id: {{ kube_oidc_client_id }}
-{%   if kube_oidc_ca_file is defined %}
-  oidc-ca-file: {{ kube_oidc_ca_file }}
-{%   endif %}
-{%   if kube_oidc_username_claim is defined %}
-  oidc-username-claim: {{ kube_oidc_username_claim }}
-{%   endif %}
-{%   if kube_oidc_groups_claim is defined %}
-  oidc-groups-claim: {{ kube_oidc_groups_claim }}
-{%   endif %}
-{%   if kube_oidc_username_prefix is defined %}
-  oidc-username-prefix: "{{ kube_oidc_username_prefix }}"
-{%   endif %}
-{%   if kube_oidc_groups_prefix is defined %}
-  oidc-groups-prefix: "{{ kube_oidc_groups_prefix }}"
-{%   endif %}
-{% endif %}
-{% if kube_webhook_token_auth|default(false) %}
-  authentication-token-webhook-config-file: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-{% endif %}
-{% if kube_encrypt_secret_data %}
-  experimental-encryption-provider-config: {{ kube_cert_dir }}/secrets_encryption.yaml
-{% endif %}
-  storage-backend: {{ kube_apiserver_storage_backend }}
-{% if kube_api_runtime_config is defined %}
-  runtime-config: {{ kube_api_runtime_config | join(',') }}
-{% endif %}
-  allow-privileged: "true"
-{% for key in kube_kubeadm_apiserver_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
-{% endfor %}
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if kube_network_plugin is defined and kube_network_plugin == 'cloud' %}
-  configure-cloud-routes: "true"
-{% endif %}
-controllerManagerExtraArgs:
-  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
-  node-monitor-period: {{ kube_controller_node_monitor_period }}
-  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
-  node-cidr-mask-size: "{{ kube_network_node_prefix }}"
-  profiling: "{{ kube_profiling }}"
-  terminated-pod-gc-threshold: "{{ kube_controller_terminated_pod_gc_threshold }}"
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% for key in kube_kubeadm_controller_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
-{% endfor %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-controllerManagerExtraVolumes:
-{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined and openstack_cacert != "" %}
-- name: openstackcacert
-  hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-  mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
-{% endif %}
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-- name: cloud-config
-  hostPath: {{ kube_config_dir }}/cloud_config
-  mountPath: {{ kube_config_dir }}/cloud_config
-{% endif %}
-{% endif %}
-schedulerExtraArgs:
-  profiling: "{{ kube_profiling }}"
-{% if kube_feature_gates %}
-  feature-gates: {{ kube_feature_gates|join(',') }}
-{% endif %}
-{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
-{% for key in kube_kubeadm_scheduler_extra_args %}
-  {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
-{% endfor %}
-{% endif %}
-{% if kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ssl_ca_dirs|length %}
-apiServerExtraVolumes:
-{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
-- name: cloud-config
-  hostPath: {{ kube_config_dir }}/cloud_config
-  mountPath: {{ kube_config_dir }}/cloud_config
-{% endif %}
-{% if kube_basic_auth|default(true) %}
-- name: basic-auth-config
-  hostPath: {{ kube_users_dir }}
-  mountPath: {{ kube_users_dir }}
-{% endif %}
-{% if kube_token_auth|default(true) %}
-- name: token-auth-config
-  hostPath: {{ kube_token_dir }}
-  mountPath: {{ kube_token_dir }}
-{% endif %}
-{% if kube_webhook_token_auth|default(false) %}
-- name: webhook-token-auth-config
-  hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-  mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
-{% endif %}
-{% if ssl_ca_dirs|length %}
-{% for dir in ssl_ca_dirs %}
-- name: {{ dir | regex_replace('^/(.*)$', '\\1' ) | regex_replace('/', '-') }}
-  hostPath: {{ dir }}
-  mountPath: {{ dir }}
-  writable: false
-{% endfor %}
-{% endif %}
-{% endif %}
-apiServerCertSANs:
-{% for san in apiserver_sans %}
-  - {{ san }}
-{% endfor %}
-certificatesDir: {{ kube_cert_dir }}
-imageRepository: {{ kube_image_repo }}
-unifiedControlPlaneImage: ""
-{% if kube_override_hostname|default('') %}
-nodeName: {{ kube_override_hostname }}
-{% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -14,9 +14,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ etcd_cert_dir }}/ca.pem
-      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
+      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
+      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -221,10 +221,12 @@ nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
-{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+{% if inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% else %}
+  taints: {}
 {% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -7,10 +7,12 @@ nodeRegistration:
 {% if kube_override_hostname|default('') %}
   name: {{ kube_override_hostname }}
 {% endif %}
-{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+{% if inventory_hostname not in groups['kube-node'] %}
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% else %}
+  taints: {}
 {% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
@@ -29,9 +31,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ etcd_cert_dir }}/ca.pem
-      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
+      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
+      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}

roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2

@@ -11,6 +11,8 @@ nodeRegistration:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% else %}
+  taints: []
 {% endif %}
 {% if container_manager == 'crio' %}
   criSocket: /var/run/crio/crio.sock
@@ -29,9 +31,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ etcd_cert_dir }}/ca.pem
-      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
-      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}
+      certFile: {{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}
+      keyFile: {{ etcd_cert_dir }}/{{ kube_etcd_key_file }}
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}