Upgrade kubernetes to v1.13.0 (#3810)

* Upgrade kubernetes to v1.13.0

* Remove all precense of scheduler.alpha.kubernetes.io/critical-pod in templates

* Fix cert dir

* Use kubespray v2.8 as baseline for gitlab

.gitlab-ci.yml

@@ -41,7 +41,7 @@ before_script:
   tags:
     - kubernetes
     - docker
-  image: quay.io/kubespray/kubespray:v2.7
+  image: quay.io/kubespray/kubespray:v2.8
 
 .docker_service: &docker_service
   services:
@@ -88,11 +88,11 @@ before_script:
     - echo ${PWD}
     - echo "${STARTUP_SCRIPT}"
     - cd tests && make create-${CI_PLATFORM} -s ; cd -
-    #- git fetch --all && git checkout v2.7.0
 
     # Check out latest tag if testing upgrade
     # Uncomment when gitlab kubespray repo has tags
-    - test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+    #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+    - test "${UPGRADE_TEST}" != "false" && git checkout 9051aa5296ef76fcff69a2e3827cef28752aa475
     # Checkout the CI vars file so it is available
     - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
     # Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021

README.md

@@ -111,7 +111,7 @@ Supported Components
 --------------------
 
 -   Core
-    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.12.3
+    -   [kubernetes](https://github.com/kubernetes/kubernetes) v1.13.0
     -   [etcd](https://github.com/coreos/etcd) v3.2.24
     -   [docker](https://www.docker.com/) v18.06 (see note)
     -   [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)

inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml

@@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
 kube_api_anonymous_auth: true
 
 ## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.12.3
+kube_version: v1.13.0
 
 # kubernetes image repo define
 kube_image_repo: "gcr.io/google-containers"

roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2

@@ -28,7 +28,6 @@ spec:
       labels:
         k8s-app: dnsmasq-autoscaler
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/download/defaults/main.yml

@@ -35,7 +35,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube
 image_arch: "{{host_architecture | default('amd64')}}"
 
 # Versions
-kube_version: v1.12.3
+kube_version: v1.13.0
 kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.24
 
@@ -70,6 +70,7 @@ cni_download_url: "https://github.com/containernetworking/plugins/releases/downl
 
 # Checksums
 hyperkube_checksums:
+  v1.13.0: 754f1baae5dc2ba29afc66e1f5d3b676ee59cd5c40ccce813092408d53bde3d9
   v1.12.3: 600aad3f0d016716abd85931239806193ffbe95f2edfdcea11532d518ae5cdb1
   v1.12.2: 566dfed398c20c9944f8999d6370cb584cb8c228b3c5881137b6b3d9306e4b06
   v1.12.1: 4aa23cfb2fc2e2e4d0cbe0d83a648c38e4baabd6c66f5cdbbb40cbc7582fdc74
@@ -88,6 +89,7 @@ hyperkube_checksums:
   v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c
   v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e
 kubeadm_checksums:
+  v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06
   v1.12.3: c675aa3be82754b3f8dfdde2a1526a72986713312d46d898e65cb564c6aa8ad4
   v1.12.2: 51bc4bfd1d934a27245111c0ad1f793d5147ed15389415a1509502f23fcfa642
   v1.12.1: 5d95efd65aad398d85a9802799f36410ae7a95f9cbe73c8b10d2213c10a6d7be

roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2

@@ -31,7 +31,6 @@ spec:
       labels:
         k8s-app: dns-autoscaler{{ coredns_ordinal_suffix | default('') }}
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2

@@ -25,7 +25,6 @@ spec:
       labels:
         k8s-app: kube-dns
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2

@@ -14,8 +14,6 @@ spec:
     metadata:
       labels:
         k8s-app: nvidia-gpu-device-plugin
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       affinity:

roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2

@@ -22,8 +22,6 @@ spec:
     metadata:
       labels:
         name: nvidia-driver-installer
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       priorityClassName: system-node-critical
       affinity:

roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2

@@ -21,7 +21,6 @@ spec:
         app.kubernetes.io/name: metrics-server
         version: {{ metrics_server_version }}
       annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
         seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
 {% if kube_version is version('v1.11.1', '>=') %}

roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2

@@ -6,8 +6,6 @@ metadata:
   labels:
     k8s-app: calico-kube-controllers
     kubernetes.io/cluster-service: "true"
-  annotations:
-    scheduler.alpha.kubernetes.io/critical-pod: ''
 spec:
   replicas: 1
   strategy:

roles/kubernetes/kubeadm/tasks/main.yml

@@ -46,7 +46,14 @@
 - name: sets kubeadm api version to v1alpha3
   set_fact:
     kubeadmConfig_api_version: v1alpha3
-  when: kubeadm_output.stdout is version('v1.12.0', '>=')
+  when:
+    - kubeadm_output.stdout is version('v1.12.0', '>=')
+    - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+  set_fact:
+    kubeadmConfig_api_version: v1beta1
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
 
 - name: Create kubeadm client config
   template:

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2

 apiVersion: kubeadm.k8s.io/v1alpha1
 kind: NodeConfiguration
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
 token: {{ kubeadm_token }}
 discoveryTokenAPIServers:
 {% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2

@@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha2
 kind: NodeConfiguration
 clusterName: {{ cluster_name }}
 discoveryFile: ""
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
 discoveryTimeout: {{ discovery_timeout }}
 discoveryToken: {{ kubeadm_token }}
 tlsBootstrapToken: {{ kubeadm_token }}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2

@@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha3
 kind: JoinConfiguration
 clusterName: {{ cluster_name }}
 discoveryFile: ""
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
 discoveryTimeout: {{ discovery_timeout }}
 discoveryToken: {{ kubeadm_token }}
 tlsBootstrapToken: {{ kubeadm_token }}

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2

+apiVersion: kubeadm.k8s.io/v1beta1
+kind: JoinConfiguration
+discovery:
+  bootstrapToken:
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+    apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+    apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
+{% endif %}
+    token: {{ kubeadm_token }}
+    unsafeSkipCAVerification: true
+  timeout: {{ discovery_timeout }}
+  tlsBootstrapToken: {{ kubeadm_token }}
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+controlPlane:
+  localAPIEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% endif %}
+caCertPath: {{ kube_cert_dir }}/ca.crt
+nodeRegistration:
+  name: {{ inventory_hostname  }}
+{% if container_manager == 'crio' %}
+  criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+  criSocket: /var/run/rkt.sock
+{% else %}
+  criSocket: /var/run/dockershim.sock
+{% endif %}

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -103,7 +103,14 @@
 - name: sets kubeadm api version to v1alpha3
   set_fact:
     kubeadmConfig_api_version: v1alpha3
-  when: kubeadm_output.stdout is version('v1.12.0', '>=')
+  when:
+    - kubeadm_output.stdout is version('v1.12.0', '>=')
+    - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+  set_fact:
+    kubeadmConfig_api_version: v1beta1
+  when: kubeadm_output.stdout is version('v1.13.0', '>=')
 
 # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
 - name: set kubeadm_config_api_fqdn define
@@ -144,15 +151,6 @@
   failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
   notify: Master | restart kubelet
 
-# FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed
-- name: kubeadm | Enable kube-proxy
-  command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml"
-  register: kubeadm_kube_proxy_enable
-  retries: 10
-  until: kubeadm_kube_proxy_enable is succeeded
-  when: inventory_hostname == groups['kube-master']|first
-  changed_when: false
-
 - name: slurp kubeadm certs
   slurp:
     src: "{{ item }}"

roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2

@@ -13,9 +13,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
   - {{ endpoint }}
 {% endfor %}
-  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
-  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
-  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+  caFile: {{ etcd_cert_dir }}/ca.pem
+  certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+  keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -69,6 +69,7 @@ apiServerExtraArgs:
 {% if kube_version is version('v1.9', '>=') %}
   endpoint-reconciler-type: lease
 {% endif %}
+  storage-backend: etcd3
 {% if etcd_events_cluster_enabled %}
   etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -14,9 +14,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
-      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
-      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ etcd_cert_dir }}/ca.pem
+      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -54,6 +54,7 @@ apiServerExtraArgs:
 {% if kube_version is version('v1.9', '>=') %}
   endpoint-reconciler-type: lease
 {% endif %}
+  storage-backend: etcd3
 {% if etcd_events_cluster_enabled %}
   etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
 {% endif %}

roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2

@@ -29,9 +29,9 @@ etcd:
 {% for endpoint in etcd_access_addresses.split(',') %}
       - {{ endpoint }}
 {% endfor %}
-      caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
-      certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
-      keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+      caFile: {{ etcd_cert_dir }}/ca.pem
+      certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+      keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -71,6 +71,7 @@ apiServerExtraArgs:
 {% if kube_version is version('v1.9', '>=') %}
   endpoint-reconciler-type: lease
 {% endif %}
+  storage-backend: etcd3
 {% if etcd_events_cluster_enabled %}
   etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
 {% endif %}