Use K8s 1.14 and add kubeadm experimental control plane mode (#4514)

* Use K8s 1.14 and add kubeadm experimental control plane mode

This reverts commit d39c273d.

* Cleanup kubeadm setup run on first master

* pin kubeadm_certificate_key in test

* Remove kubelet autolabel of kube-node, add symlink for pki dir

Change-Id: Id5e74dd667c60675dbfe4193b0bc9fb44380e1ca

roles/kubernetes/master/templates/kubeadm-controlplane.v1beta1.yaml.j2

+apiVersion: kubeadm.k8s.io/v1beta1
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+{% if kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
+{% endif %}
+    token: {{ kubeadm_token }}
+    unsafeSkipCAVerification: true
+  timeout: {{ discovery_timeout }}
+  tlsBootstrapToken: {{ kubeadm_token }}
+controlPlane:
+  localAPIEndpoint:
+    advertiseAddress: {{ kube_apiserver_address }}
+    bindPort: {{ kube_apiserver_port }}
+nodeRegistration:
+  name: {{ inventory_hostname  }}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
+{% endif %}

roles/kubernetes/node/templates/kubelet.kubeadm.env.j2

@@ -84,14 +84,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 
 {# Kubelet node labels #}
 {% set role_node_labels = [] %}
-{% if inventory_hostname in groups['kube-master'] %}
-{%   set dummy = role_node_labels.append("node-role.kubernetes.io/master=''") %}
-{%   if not standalone_kubelet|bool %}
-{%     set dummy = role_node_labels.append("node-role.kubernetes.io/node=''") %}
-{%   endif %}
-{% else %}
-{%   set dummy = role_node_labels.append("node-role.kubernetes.io/node=''") %}
-{% endif %}
 {% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
 {%   if inventory_hostname in nvidia_gpu_nodes %}
 {%     set dummy = role_node_labels.append('nvidia.com/gpu=true')  %}

roles/kubernetes/preinstall/defaults/main.yml

@@ -24,6 +24,8 @@ disable_ipv6_dns: false
 
 kube_cert_group: kube-cert
 kube_config_dir: /etc/kubernetes
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
+kube_cert_compat_dir: /etc/kubernetes/pki
 
 # Container Linux by CoreOS cloud init config file to define /etc/resolv.conf content
 # for hostnet pods and infra needs

roles/kubernetes/preinstall/tasks/0050-create_directories.yml

@@ -23,6 +23,14 @@
     - "{{ kube_manifest_dir }}"
     - "{{ kube_script_dir }}"
 
+- name: Create kubernetes kubeadm compat cert dir (kubernetes/kubeadm issue 1498)
+  file:
+    src: "{{ kube_cert_dir }}"
+    dest: "{{ kube_cert_compat_dir }}"
+    state: link
+  when:
+    - kube_cert_dir != kube_cert_compat_dir
+
 - name: Create cni directories
   file:
     path: "{{ item }}"

roles/kubespray-defaults/defaults/main.yaml

@@ -12,7 +12,7 @@ is_atomic: false
 disable_swap: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.13.5
+kube_version: v1.14.0
 
 ## Kube Proxy mode One of ['iptables','ipvs']
 kube_proxy_mode: ipvs
@@ -97,6 +97,9 @@ kube_manifest_dir: "{{ kube_config_dir }}/manifests"
 # This is where all the cert scripts and certs will be located
 kube_cert_dir: "{{ kube_config_dir }}/ssl"
 
+# compatibility directory for kubeadm
+kube_cert_compat_dir: "/etc/kubernetes/pki"
+
 # This is where all of the bearer tokens will be stored
 kube_token_dir: "{{ kube_config_dir }}/tokens"
 
@@ -335,6 +338,9 @@ kube_feature_gates: |-
   {{ feature_gate_v1_12 }}
   {%- endif %}
 
+# Enable kubeadm experimental control plane
+kubeadm_control_plane: false
+
 # Local volume provisioner storage classes
 # Levarages Ansibles string to Python datatype casting. Otherwise the dict_key isn't substituted
 # see https://github.com/ansible/ansible/issues/17324
@@ -383,7 +389,7 @@ no_proxy: >-
   {%- endif -%}
   {%- for item in (groups['k8s-cluster'] + groups['etcd'] + groups['calico-rr']|default([]))|unique -%}
   {{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(fallback_ips[item])) }},
-  {%-   if item != hostvars[item].get('ansible_hostname', "") -%}
+  {%-   if item != hostvars[item].get('ansible_hostname', '') -%}
   {{ hostvars[item]['ansible_hostname'] }},
   {{ hostvars[item]['ansible_hostname'] }}.{{ dns_domain }},
   {%-   endif -%}

roles/network_plugin/calico/defaults/main.yml

@@ -61,3 +61,7 @@ calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostna
 
 ### do not enable this, this is detected in scope of tasks, this is just a default value
 calico_upgrade_needed: false
+
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem

roles/network_plugin/calico/handlers/main.yml

 ---
-- name: restart calico-node
+- name: reset_calico_cni
   command: /bin/true
   notify:
-    - Calico | reload systemd
-    - Calico | reload calico-node
+    - delete 10-calico.conflist
+    - delete calico-node containers
 
-- name: Calico | reload systemd
-  shell: systemctl daemon-reload
+- name: delete 10-calico.conflist
+  file:
+    path: /etc/calico/10-calico.conflist
+    state: absent
 
-- name: Calico | reload calico-node
-  service:
-    name: calico-node
-    state: restarted
-    sleep: 10
+- name: delete calico-node containers
+  shell: "docker ps -af name=k8s_POD_calico-node* -q | xargs --no-run-if-empty docker rm -f"

roles/network_plugin/calico/rr/defaults/main.yml

@@ -10,3 +10,7 @@ calico_rr_memory_limit: 1000M
 calico_rr_cpu_limit: 300m
 calico_rr_memory_requests: 128M
 calico_rr_cpu_requests: 150m
+
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem

roles/network_plugin/calico/rr/tasks/main.yml

@@ -22,9 +22,9 @@
     state: hard
     force: yes
   with_items:
-    - {s: "ca.pem", d: "ca_cert.crt"}
-    - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
-    - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
 
 - name: Calico-rr | Create dir for logs
   file:

roles/network_plugin/calico/tasks/install.yml

@@ -11,6 +11,8 @@
     src: "cni-calico.conflist.j2"
     dest: "/etc/cni/net.d/{% if calico_version is version('v3.3.0', '>=') %}calico.conflist.template{% else %}10-calico.conflist{% endif %}"
     owner: kube
+  register: calico_conflist
+  notify: reset_calico_cni
 
 - name: Calico | Create calico certs directory
   file:
@@ -27,9 +29,9 @@
     state: hard
     force: yes
   with_items:
-    - {s: "ca.pem", d: "ca_cert.crt"}
-    - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
-    - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
 
 - name: Calico | Install calicoctl wrapper script
   template:

roles/network_plugin/calico/templates/etcdv2-store.yml.j2

@@ -4,6 +4,6 @@ metadata:
 spec:
   datastoreType: "etcdv2"
   etcdEndpoints: "{{ etcd_access_addresses }}"
-  etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
-  etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
-  etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem"
+  etcdKeyFile: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}"
+  etcdCertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}"
+  etcdCACertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}"

roles/network_plugin/calico/templates/etcdv3-store.yml.j2

@@ -4,6 +4,6 @@ metadata:
 spec:
   datastoreType: "etcdv3"
   etcdEndpoints: "{{ etcd_access_addresses }}"
-  etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
-  etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
-  etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem"
\ No newline at end of file
+  etcdKeyFile: "{{ etcd_cert_dir }}/{{ kube_etcd_key_file }}"
+  etcdCertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cert_file }}"
+  etcdCACertFile: "{{ etcd_cert_dir }}/{{ kube_etcd_cacert_file }}"

roles/network_plugin/canal/defaults/main.yml

@@ -30,3 +30,8 @@ calicoctl_memory_limit: 170M
 calicoctl_cpu_limit: 100m
 calicoctl_memory_requests: 32M
 calicoctl_cpu_requests: 25m
+
+# etcd cert filenames
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem

roles/network_plugin/canal/tasks/main.yml

@@ -20,9 +20,9 @@
     state: hard
     force: yes
   with_items:
-    - {s: "ca.pem", d: "ca_cert.crt"}
-    - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
-    - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
 
 - name: Canal | Set Flannel etcd configuration
   command: |-

roles/network_plugin/cilium/defaults/main.yml

@@ -5,6 +5,9 @@ cilium_disable_ipv4: false
 
 # Etcd SSL dirs
 cilium_cert_dir: /etc/cilium/certs
+kube_etcd_cacert_file: ca.pem
+kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
+kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
 
 # Cilium Network Policy directory
 cilium_policy_dir: /etc/kubernetes/policy

roles/network_plugin/cilium/tasks/main.yml

@@ -21,9 +21,9 @@
     state: hard
     force: yes
   with_items:
-    - {s: "ca.pem", d: "ca_cert.crt"}
-    - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
-    - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
+    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
+    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
+    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
 
 - name: Cilium | Create Cilium node manifests
   template:

tests/files/gce_centos7-flannel-addons.yml

@@ -6,6 +6,8 @@ cloud_machine_type: "n1-standard-2"
 mode: ha
 
 # Deployment settings
+kubeadm_control_plane: true
+kubeadm_certificate_key: 3998c58db6497dd17d909394e62d515368c06ec617710d02edea31c06d741085
 kube_network_plugin: flannel
 helm_enabled: true
 kubernetes_audit: true