From 225f765b56d21df202a2b932f73b71854527dad9 Mon Sep 17 00:00:00 2001
From: Rong Zhang <rongzhang@alauda.io>
Date: Fri, 7 Dec 2018 04:11:48 +0800
Subject: [PATCH] Upgrade kubernetes to v1.13.0 (#3810)
* Upgrade kubernetes to v1.13.0
* Remove all precense of scheduler.alpha.kubernetes.io/critical-pod in templates
* Fix cert dir
* Use kubespray v2.8 as baseline for gitlab
---
.gitlab-ci.yml | 6 +-
README.md | 2 +-
.../group_vars/k8s-cluster/k8s-cluster.yml | 2 +-
.../templates/dnsmasq-autoscaler.yml.j2 | 1 -
roles/download/defaults/main.yml | 4 +-
.../ansible/templates/dns-autoscaler.yml.j2 | 1 -
.../ansible/templates/kubedns-deploy.yml.j2 | 1 -
.../k8s-device-plugin-nvidia-daemonset.yml.j2 | 2 -
.../nvidia-driver-install-daemonset.yml.j2 | 2 -
.../metrics-server-deployment.yaml.j2 | 1 -
.../templates/calico-kube-controllers.yml.j2 | 2 -
roles/kubernetes/kubeadm/tasks/main.yml | 9 +-
.../templates/kubeadm-client.conf.v1alpha1.j2 | 2 +-
.../templates/kubeadm-client.conf.v1alpha2.j2 | 2 +-
.../templates/kubeadm-client.conf.v1alpha3.j2 | 2 +-
.../templates/kubeadm-client.conf.v1beta1.j2 | 27 ++
.../kubernetes/master/tasks/kubeadm-setup.yml | 18 +-
.../templates/kubeadm-config.v1alpha1.yaml.j2 | 7 +-
.../templates/kubeadm-config.v1alpha2.yaml.j2 | 7 +-
.../templates/kubeadm-config.v1alpha3.yaml.j2 | 7 +-
.../templates/kubeadm-config.v1beta1.yaml.j2 | 258 ++++++++++++++++++
roles/kubespray-defaults/defaults/main.yaml | 2 +-
.../calico/templates/calico-node.yml.j2 | 1 -
.../canal/templates/canal-node.yaml.j2 | 3 -
.../cilium/templates/cilium-ds.yml.j2 | 6 -
.../contiv/templates/contiv-api-proxy.yml.j2 | 3 -
.../contiv/templates/contiv-cleanup.yml.j2 | 3 -
.../contiv/templates/contiv-etcd-proxy.yml.j2 | 2 -
.../contiv/templates/contiv-etcd.yml.j2 | 2 -
.../contiv/templates/contiv-netmaster.yml.j2 | 3 -
.../contiv/templates/contiv-netplugin.yml.j2 | 3 -
.../contiv/templates/contiv-ovs.yml.j2 | 3 -
.../flannel/templates/cni-flannel.yml.j2 | 3 -
.../kube-router/templates/kube-router.yml.j2 | 2 -
.../weave/templates/weave-net.yml.j2 | 3 -
35 files changed, 325 insertions(+), 77 deletions(-)
create mode 100644 roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2
create mode 100644 roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 845f28ca1..21abfb629 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -41,7 +41,7 @@ before_script:
tags:
- kubernetes
- docker
- image: quay.io/kubespray/kubespray:v2.7
+ image: quay.io/kubespray/kubespray:v2.8
.docker_service: &docker_service
services:
@@ -88,11 +88,11 @@ before_script:
- echo ${PWD}
- echo "${STARTUP_SCRIPT}"
- cd tests && make create-${CI_PLATFORM} -s ; cd -
- #- git fetch --all && git checkout v2.7.0
# Check out latest tag if testing upgrade
# Uncomment when gitlab kubespray repo has tags
- - test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+ #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
+ - test "${UPGRADE_TEST}" != "false" && git checkout 9051aa5296ef76fcff69a2e3827cef28752aa475
# Checkout the CI vars file so it is available
- test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
# Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021
diff --git a/README.md b/README.md
index 1138419b2..67d98bd58 100644
--- a/README.md
+++ b/README.md
@@ -111,7 +111,7 @@ Supported Components
--------------------
- Core
- - [kubernetes](https://github.com/kubernetes/kubernetes) v1.12.3
+ - [kubernetes](https://github.com/kubernetes/kubernetes) v1.13.0
- [etcd](https://github.com/coreos/etcd) v3.2.24
- [docker](https://www.docker.com/) v18.06 (see note)
- [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)
diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml
index ffd158859..e1a800327 100644
--- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml
@@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users"
kube_api_anonymous_auth: true
## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.12.3
+kube_version: v1.13.0
# kubernetes image repo define
kube_image_repo: "gcr.io/google-containers"
diff --git a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2
index ec7e43fdb..1b81f4c48 100644
--- a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2
+++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2
@@ -28,7 +28,6 @@ spec:
labels:
k8s-app: dnsmasq-autoscaler
annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
spec:
{% if kube_version is version('v1.11.1', '>=') %}
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 3bc9e0134..8dd6ae576 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -35,7 +35,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube
image_arch: "{{host_architecture | default('amd64')}}"
# Versions
-kube_version: v1.12.3
+kube_version: v1.13.0
kubeadm_version: "{{ kube_version }}"
etcd_version: v3.2.24
@@ -70,6 +70,7 @@ cni_download_url: "https://github.com/containernetworking/plugins/releases/downl
# Checksums
hyperkube_checksums:
+ v1.13.0: 754f1baae5dc2ba29afc66e1f5d3b676ee59cd5c40ccce813092408d53bde3d9
v1.12.3: 600aad3f0d016716abd85931239806193ffbe95f2edfdcea11532d518ae5cdb1
v1.12.2: 566dfed398c20c9944f8999d6370cb584cb8c228b3c5881137b6b3d9306e4b06
v1.12.1: 4aa23cfb2fc2e2e4d0cbe0d83a648c38e4baabd6c66f5cdbbb40cbc7582fdc74
@@ -88,6 +89,7 @@ hyperkube_checksums:
v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c
v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e
kubeadm_checksums:
+ v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06
v1.12.3: c675aa3be82754b3f8dfdde2a1526a72986713312d46d898e65cb564c6aa8ad4
v1.12.2: 51bc4bfd1d934a27245111c0ad1f793d5147ed15389415a1509502f23fcfa642
v1.12.1: 5d95efd65aad398d85a9802799f36410ae7a95f9cbe73c8b10d2213c10a6d7be
diff --git a/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 b/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
index 2d930c6e3..20c86550d 100644
--- a/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2
@@ -31,7 +31,6 @@ spec:
labels:
k8s-app: dns-autoscaler{{ coredns_ordinal_suffix | default('') }}
annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
{% if kube_version is version('v1.11.1', '>=') %}
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
index fa2507115..a095f1625 100644
--- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2
@@ -25,7 +25,6 @@ spec:
labels:
k8s-app: kube-dns
annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
{% if kube_version is version('v1.11.1', '>=') %}
diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2
index 84f440442..5fd84012c 100644
--- a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2
+++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2
@@ -14,8 +14,6 @@ spec:
metadata:
labels:
k8s-app: nvidia-gpu-device-plugin
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
priorityClassName: system-node-critical
affinity:
diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2
index a1adede5a..5dda82189 100644
--- a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2
+++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2
@@ -22,8 +22,6 @@ spec:
metadata:
labels:
name: nvidia-driver-installer
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
priorityClassName: system-node-critical
affinity:
diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2
index 6cb51d025..245c3beab 100644
--- a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2
+++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2
@@ -21,7 +21,6 @@ spec:
app.kubernetes.io/name: metrics-server
version: {{ metrics_server_version }}
annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
{% if kube_version is version('v1.11.1', '>=') %}
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
index dcb7c9f5f..1eb8e1d4d 100644
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
@@ -6,8 +6,6 @@ metadata:
labels:
k8s-app: calico-kube-controllers
kubernetes.io/cluster-service: "true"
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
replicas: 1
strategy:
diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
index 29b866a97..c2035859d 100644
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -46,7 +46,14 @@
- name: sets kubeadm api version to v1alpha3
set_fact:
kubeadmConfig_api_version: v1alpha3
- when: kubeadm_output.stdout is version('v1.12.0', '>=')
+ when:
+ - kubeadm_output.stdout is version('v1.12.0', '>=')
+ - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+ set_fact:
+ kubeadmConfig_api_version: v1beta1
+ when: kubeadm_output.stdout is version('v1.13.0', '>=')
- name: Create kubeadm client config
template:
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2
index fe9f45b2f..6a40ab03e 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2
@@ -1,6 +1,6 @@
apiVersion: kubeadm.k8s.io/v1alpha1
kind: NodeConfiguration
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
token: {{ kubeadm_token }}
discoveryTokenAPIServers:
{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
index eebcdf7c0..b5d8365d7 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2
@@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha2
kind: NodeConfiguration
clusterName: {{ cluster_name }}
discoveryFile: ""
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
discoveryTimeout: {{ discovery_timeout }}
discoveryToken: {{ kubeadm_token }}
tlsBootstrapToken: {{ kubeadm_token }}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
index a1e0887e0..ff5f2c0b4 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
@@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha3
kind: JoinConfiguration
clusterName: {{ cluster_name }}
discoveryFile: ""
-caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+caCertPath: {{ kube_cert_dir }}/ca.crt
discoveryTimeout: {{ discovery_timeout }}
discoveryToken: {{ kubeadm_token }}
tlsBootstrapToken: {{ kubeadm_token }}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2
new file mode 100644
index 000000000..2eb0cf368
--- /dev/null
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2
@@ -0,0 +1,27 @@
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: JoinConfiguration
+discovery:
+ bootstrapToken:
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+ apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+ apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}}
+{% endif %}
+ token: {{ kubeadm_token }}
+ unsafeSkipCAVerification: true
+ timeout: {{ discovery_timeout }}
+ tlsBootstrapToken: {{ kubeadm_token }}
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+controlPlane:
+ localAPIEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% endif %}
+caCertPath: {{ kube_cert_dir }}/ca.crt
+nodeRegistration:
+ name: {{ inventory_hostname }}
+{% if container_manager == 'crio' %}
+ criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+ criSocket: /var/run/rkt.sock
+{% else %}
+ criSocket: /var/run/dockershim.sock
+{% endif %}
diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index c826d9c71..1ed271636 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -103,7 +103,14 @@
- name: sets kubeadm api version to v1alpha3
set_fact:
kubeadmConfig_api_version: v1alpha3
- when: kubeadm_output.stdout is version('v1.12.0', '>=')
+ when:
+ - kubeadm_output.stdout is version('v1.12.0', '>=')
+ - kubeadm_output.stdout is version('v1.13.0', '<')
+
+- name: sets kubeadm api version to v1beta1
+ set_fact:
+ kubeadmConfig_api_version: v1beta1
+ when: kubeadm_output.stdout is version('v1.13.0', '>=')
# Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
- name: set kubeadm_config_api_fqdn define
@@ -144,15 +151,6 @@
failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr
notify: Master | restart kubelet
-# FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed
-- name: kubeadm | Enable kube-proxy
- command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml"
- register: kubeadm_kube_proxy_enable
- retries: 10
- until: kubeadm_kube_proxy_enable is succeeded
- when: inventory_hostname == groups['kube-master']|first
- changed_when: false
-
- name: slurp kubeadm certs
slurp:
src: "{{ item }}"
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
index ee780cd5e..f2ad127c7 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2
@@ -13,9 +13,9 @@ etcd:
{% for endpoint in etcd_access_addresses.split(',') %}
- {{ endpoint }}
{% endfor %}
- caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
- certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
- keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+ caFile: {{ etcd_cert_dir }}/ca.pem
+ certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+ keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
networking:
dnsDomain: {{ dns_domain }}
serviceSubnet: {{ kube_service_addresses }}
@@ -69,6 +69,7 @@ apiServerExtraArgs:
{% if kube_version is version('v1.9', '>=') %}
endpoint-reconciler-type: lease
{% endif %}
+ storage-backend: etcd3
{% if etcd_events_cluster_enabled %}
etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
{% endif %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
index 87e5a961e..3385d2892 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
@@ -14,9 +14,9 @@ etcd:
{% for endpoint in etcd_access_addresses.split(',') %}
- {{ endpoint }}
{% endfor %}
- caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
- certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
- keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+ caFile: {{ etcd_cert_dir }}/ca.pem
+ certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+ keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
networking:
dnsDomain: {{ dns_domain }}
serviceSubnet: {{ kube_service_addresses }}
@@ -54,6 +54,7 @@ apiServerExtraArgs:
{% if kube_version is version('v1.9', '>=') %}
endpoint-reconciler-type: lease
{% endif %}
+ storage-backend: etcd3
{% if etcd_events_cluster_enabled %}
etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
{% endif %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
index 13053ae0b..d6f77ff7f 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2
@@ -29,9 +29,9 @@ etcd:
{% for endpoint in etcd_access_addresses.split(',') %}
- {{ endpoint }}
{% endfor %}
- caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
- certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
- keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+ caFile: {{ etcd_cert_dir }}/ca.pem
+ certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+ keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
networking:
dnsDomain: {{ dns_domain }}
serviceSubnet: {{ kube_service_addresses }}
@@ -71,6 +71,7 @@ apiServerExtraArgs:
{% if kube_version is version('v1.9', '>=') %}
endpoint-reconciler-type: lease
{% endif %}
+ storage-backend: etcd3
{% if etcd_events_cluster_enabled %}
etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
{% endif %}
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
new file mode 100644
index 000000000..366cbee23
--- /dev/null
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2
@@ -0,0 +1,258 @@
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: InitConfiguration
+localAPIEndpoint:
+ advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+ bindPort: {{ kube_apiserver_port }}
+nodeRegistration:
+{% if kube_override_hostname|default('') %}
+ name: {{ kube_override_hostname }}
+{% endif %}
+{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %}
+ taints:
+ - key: "kubeadmNode"
+ value: "master"
+ effect: "NoSchedule"
+{% endif %}
+{% if container_manager == 'crio' %}
+ criSocket: /var/run/crio/crio.sock
+{% elif container_manager == 'rkt' %}
+ criSocket: /var/run/rkt.sock
+{% else %}
+ criSocket: /var/run/dockershim.sock
+{% endif %}
+---
+apiVersion: kubeadm.k8s.io/v1beta1
+kind: ClusterConfiguration
+clusterName: {{ cluster_name }}
+etcd:
+ external:
+ endpoints:
+{% for endpoint in etcd_access_addresses.split(',') %}
+ - {{ endpoint }}
+{% endfor %}
+ caFile: {{ etcd_cert_dir }}/ca.pem
+ certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+ keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
+networking:
+ dnsDomain: {{ dns_domain }}
+ serviceSubnet: {{ kube_service_addresses }}
+ podSubnet: {{ kube_pods_subnet }}
+ podNetworkCidr: "{{ kube_network_node_prefix }}"
+kubernetesVersion: {{ kube_version }}
+{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %}
+controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
+{% else %}
+controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }}
+{% endif %}
+certificatesDir: {{ kube_cert_dir }}
+imageRepository: {{ kube_image_repo }}
+UseHyperKubeImage: false
+apiServer:
+ extraArgs:
+ authorization-mode: {{ authorization_modes | join(',') }}
+ bind-address: {{ kube_apiserver_bind_address }}
+{% if kube_apiserver_insecure_port|string != "0" %}
+ insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+{% endif %}
+ insecure-port: "{{ kube_apiserver_insecure_port }}"
+{% if kube_version is version('v1.10', '<') %}
+ admission-control: {{ kube_apiserver_admission_control | join(',') }}
+{% else %}
+{% if kube_apiserver_enable_admission_plugins|length > 0 %}
+ enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
+{% endif %}
+{% if kube_apiserver_disable_admission_plugins|length > 0 %}
+ disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
+{% endif %}
+{% endif %}
+ apiserver-count: "{{ kube_apiserver_count }}"
+{% if kube_version is version('v1.9', '>=') %}
+ endpoint-reconciler-type: lease
+{% endif %}
+ storage-backend: etcd3
+{% if etcd_events_cluster_enabled %}
+ etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}"
+{% endif %}
+ service-node-port-range: {{ kube_apiserver_node_port_range }}
+ kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}"
+{% if kube_basic_auth|default(true) %}
+ basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_token_auth|default(true) %}
+ token-auth-file: {{ kube_token_dir }}/known_tokens.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+ oidc-issuer-url: {{ kube_oidc_url }}
+ oidc-client-id: {{ kube_oidc_client_id }}
+{% if kube_oidc_ca_file is defined %}
+ oidc-ca-file: {{ kube_oidc_ca_file }}
+{% endif %}
+{% if kube_oidc_username_claim is defined %}
+ oidc-username-claim: {{ kube_oidc_username_claim }}
+{% endif %}
+{% if kube_oidc_groups_claim is defined %}
+ oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{% endif %}
+{% endif %}
+{% if kube_encrypt_secret_data %}
+ encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml
+{% endif %}
+ storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+ runtime-config: {{ kube_api_runtime_config | join(',') }}
+{% endif %}
+ allow-privileged: "true"
+{% if kubernetes_audit %}
+ audit-log-path: "{{ audit_log_path }}"
+ audit-log-maxage: "{{ audit_log_maxage }}"
+ audit-log-maxbackup: "{{ audit_log_maxbackups }}"
+ audit-log-maxsize: "{{ audit_log_maxsize }}"
+ audit-policy-file: {{ audit_policy_file }}
+{% endif %}
+{% for key in kube_kubeadm_apiserver_extra_args %}
+ {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
+{% endfor %}
+{% if kube_feature_gates %}
+ feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+ cloud-provider: {{cloud_provider}}
+ cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+ cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes %}
+ extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+ - name: cloud-config
+ hostPath: {{ kube_config_dir }}/cloud_config
+ mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if kube_basic_auth|default(true) %}
+ - name: basic-auth-config
+ hostPath: {{ kube_users_dir }}
+ mountPath: {{ kube_users_dir }}
+{% endif %}
+{% if kube_token_auth|default(true) %}
+ - name: token-auth-config
+ hostPath: {{ kube_token_dir }}
+ mountPath: {{ kube_token_dir }}
+{% endif %}
+{% if kubernetes_audit %}
+ - name: {{ audit_policy_name }}
+ hostPath: {{ audit_policy_hostpath }}
+ mountPath: {{ audit_policy_mountpath }}
+{% if audit_log_path != "-" %}
+ - name: {{ audit_log_name }}
+ hostPath: {{ audit_log_hostpath }}
+ mountPath: {{ audit_log_mountpath }}
+ writable: true
+{% endif %}
+{% endif %}
+{% for volume in apiserver_extra_volumes %}
+ - name: {{ volume.name }}
+ hostPath: {{ volume.hostPath }}
+ mountPath: {{ volume.mountPath }}
+ writable: {{ volume.writable | default(false)}}
+{% endfor %}
+{% endif %}
+ certSANs:
+{% for san in apiserver_sans.split(' ') | unique %}
+ - {{ san }}
+{% endfor %}
+ timeoutForControlPlane: 5m0s
+controllerManager:
+ extraArgs:
+ node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+ node-monitor-period: {{ kube_controller_node_monitor_period }}
+ pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kube_feature_gates %}
+ feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% for key in kube_kubeadm_controller_extra_args %}
+ {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}"
+{% endfor %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
+ cloud-provider: {{cloud_provider}}
+ cloud-config: {{ kube_config_dir }}/cloud_config
+{% elif cloud_provider is defined and cloud_provider in ["external"] %}
+ cloud-config: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
+ extraVolumes:
+{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
+ - name: openstackcacert
+ hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+ mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
+{% endif %}
+{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
+ - name: cloud-config
+ hostPath: {{ kube_config_dir }}/cloud_config
+ mountPath: {{ kube_config_dir }}/cloud_config
+{% endif %}
+{% for volume in controller_manager_extra_volumes %}
+ - name: {{ volume.name }}
+ hostPath: {{ volume.hostPath }}
+ mountPath: {{ volume.mountPath }}
+ writable: {{ volume.writable | default(false)}}
+{% endfor %}
+{% endif %}
+scheduler:
+ extraArgs:
+{% if kube_feature_gates %}
+ feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+{% if kube_kubeadm_scheduler_extra_args|length > 0 %}
+{% for key in kube_kubeadm_scheduler_extra_args %}
+ {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}"
+{% endfor %}
+{% endif %}
+ extraVolumes:
+{% if scheduler_extra_volumes %}
+ extraVolumes:
+{% for volume in scheduler_extra_volumes %}
+ - name: {{ volume.name }}
+ hostPath: {{ volume.hostPath }}
+ mountPath: {{ volume.mountPath }}
+ writable: {{ volume.writable | default(false)}}
+{% endfor %}
+{% endif %}
+---
+apiVersion: kubeproxy.config.k8s.io/v1alpha1
+kind: KubeProxyConfiguration
+bindAddress: 0.0.0.0
+clientConnection:
+ acceptContentTypes: ""
+ burst: 10
+ contentType: application/vnd.kubernetes.protobuf
+ kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
+ qps: 5
+clusterCIDR: ""
+configSyncPeriod: 15m0s
+conntrack:
+ max: null
+ maxPerCore: 32768
+ min: 131072
+ tcpCloseWaitTimeout: 1h0m0s
+ tcpEstablishedTimeout: 24h0m0s
+enableProfiling: false
+healthzBindAddress: 0.0.0.0:10256
+iptables:
+ masqueradeAll: false
+ masqueradeBit: 14
+ minSyncPeriod: 0s
+ syncPeriod: 30s
+ipvs:
+ excludeCIDRs: null
+ minSyncPeriod: 0s
+ scheduler: ""
+ syncPeriod: 30s
+metricsBindAddress: 127.0.0.1:10249
+mode: {{ kube_proxy_mode }}
+{% if kube_proxy_nodeport_addresses %}
+nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}]
+{% endif %}
+oomScoreAdj: -999
+portRange: ""
+resourceContainer: ""
+udpIdleTimeout: 250ms
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 50d058626..745e2a9f8 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -12,7 +12,7 @@ is_atomic: false
disable_swap: true
## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.12.3
+kube_version: v1.13.0
## Kube Proxy mode One of ['iptables','ipvs']
kube_proxy_mode: ipvs
diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2
index 2090372fb..d8a433679 100644
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@@ -19,7 +19,6 @@ spec:
k8s-app: calico-node
annotations:
# Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
{% if calico_felix_prometheusmetricsenabled %}
prometheus.io/scrape: 'true'
diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2
index 68a6b9910..a46608de8 100644
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@@ -12,9 +12,6 @@ spec:
k8s-app: canal-node
template:
metadata:
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
labels:
k8s-app: canal-node
spec:
diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
index b8a6306cf..bda6000ae 100755
--- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@@ -21,12 +21,6 @@ spec:
labels:
k8s-app: cilium
kubernetes.io/cluster-service: "true"
- annotations:
- # This annotation plus the CriticalAddonsOnly toleration makes
- # cilium to be a critical pod in the cluster, which ensures cilium
- # gets priority scheduling.
- # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
- scheduler.alpha.kubernetes.io/critical-pod: ''
{% if cilium_enable_prometheus %}
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
diff --git a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
index 73a7688ef..c1604d0b5 100644
--- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
@@ -15,9 +15,6 @@ spec:
namespace: kube-system
labels:
k8s-app: contiv-api-proxy
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
index 73111072c..c8de9d297 100644
--- a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
@@ -14,9 +14,6 @@ spec:
metadata:
labels:
k8s-app: contiv-cleanup
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2
index 0b67945e5..a16ee5755 100644
--- a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2
@@ -14,8 +14,6 @@ spec:
metadata:
labels:
k8s-app: contiv-etcd-proxy
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
index b8fba9cc6..e320f5b24 100644
--- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
@@ -14,8 +14,6 @@ spec:
metadata:
labels:
k8s-app: contiv-etcd
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
index e61510253..a39938f77 100644
--- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
@@ -15,9 +15,6 @@ spec:
namespace: kube-system
labels:
k8s-app: contiv-netmaster
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
index 045b9e7eb..8b2e65ebd 100644
--- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
@@ -19,9 +19,6 @@ spec:
metadata:
labels:
k8s-app: contiv-netplugin
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
index 40c37b6ad..2ec15fc82 100644
--- a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
@@ -16,9 +16,6 @@ spec:
metadata:
labels:
k8s-app: contiv-ovs
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
index 578409d02..bcaae4a6d 100644
--- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
+++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
@@ -51,9 +51,6 @@ spec:
labels:
tier: node
k8s-app: flannel
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
diff --git a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 b/roles/network_plugin/kube-router/templates/kube-router.yml.j2
index ac1029a0e..37f03ea26 100644
--- a/roles/network_plugin/kube-router/templates/kube-router.yml.j2
+++ b/roles/network_plugin/kube-router/templates/kube-router.yml.j2
@@ -60,8 +60,6 @@ spec:
labels:
k8s-app: kube-router
tier: node
- annotations:
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-cluster-critical
diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2
index 204e3f993..3d66c043c 100644
--- a/roles/network_plugin/weave/templates/weave-net.yml.j2
+++ b/roles/network_plugin/weave/templates/weave-net.yml.j2
@@ -114,9 +114,6 @@ items:
metadata:
labels:
name: weave-net
- annotations:
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-node-critical
--
GitLab