From 53d87e53c5899d4ea2904ab7e3883708dd6363d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20de=20Saint=20Martin?=
<cedric@desaintmartin.fr>
Date: Thu, 27 Sep 2018 14:28:54 +0200
Subject: [PATCH] All CNIs: support ANY toleration. (#3391)
Before, Nodes tainted with NoExecute policy did not have calico/weave Pod.
Network pod should run on all nodes whatever happens on a specific node.
Also always set the Pods to be critical.
Also remove deprecated scheduler.alpha.kubernetes.io/tolerations annotations.
---
.../network_plugin/calico/templates/calico-node.yml.j2 | 9 ++++++---
.../network_plugin/canal/templates/canal-node.yaml.j2 | 8 +++++---
roles/network_plugin/cilium/templates/cilium-ds.yml.j2 | 10 ++--------
.../contiv/templates/contiv-api-proxy.yml.j2 | 7 +++++--
.../contiv/templates/contiv-cleanup.yml.j2 | 9 +++++++--
.../network_plugin/contiv/templates/contiv-etcd.yml.j2 | 6 ++++--
.../contiv/templates/contiv-netmaster.yml.j2 | 7 +++++--
.../contiv/templates/contiv-netplugin.yml.j2 | 7 +++++--
.../network_plugin/contiv/templates/contiv-ovs.yml.j2 | 7 +++++--
.../flannel/templates/cni-flannel.yml.j2 | 10 +++++++---
roles/network_plugin/weave/templates/weave-net.yml.j2 | 9 +++++++--
11 files changed, 58 insertions(+), 31 deletions(-)
diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2
index 539ced8a6..c692bc925 100644
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@@ -18,6 +18,7 @@ spec:
labels:
k8s-app: calico-node
annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
scheduler.alpha.kubernetes.io/critical-pod: ''
kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
spec:
@@ -27,8 +28,10 @@ spec:
hostNetwork: true
serviceAccountName: calico-node
tolerations:
- - effect: NoSchedule
- operator: Exists
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
terminationGracePeriodSeconds: 0
@@ -189,4 +192,4 @@ spec:
updateStrategy:
rollingUpdate:
maxUnavailable: {{ serial | default('20%') }}
- type: RollingUpdate
\ No newline at end of file
+ type: RollingUpdate
diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2
index ea34dfa89..e0d0c7cff 100644
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@@ -13,8 +13,8 @@ spec:
template:
metadata:
annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
scheduler.alpha.kubernetes.io/critical-pod: ''
- scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
labels:
k8s-app: canal-node
spec:
@@ -24,8 +24,10 @@ spec:
hostNetwork: true
serviceAccountName: canal
tolerations:
- - effect: NoSchedule
- operator: Exists
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
volumes:
# Used by calico/node.
- name: lib-modules
diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
index 4eff22269..ff76d6d7c 100755
--- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@@ -27,8 +27,6 @@ spec:
# gets priority scheduling.
# https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
scheduler.alpha.kubernetes.io/critical-pod: ''
- scheduler.alpha.kubernetes.io/tolerations: >-
- [{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]
{% if cilium_enable_prometheus %}
prometheus.io/scrape: "true"
prometheus.io/port: "9090"
@@ -225,11 +223,7 @@ spec:
restartPolicy: Always
tolerations:
- - effect: NoSchedule
- key: node-role.kubernetes.io/master
- - effect: NoSchedule
- key: node.cloudprovider.kubernetes.io/uninitialized
- value: "true"
- # Mark cilium's pod as critical for rescheduling
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- key: CriticalAddonsOnly
operator: "Exists"
diff --git a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
index f37e83847..706027623 100644
--- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2
@@ -16,6 +16,7 @@ spec:
labels:
k8s-app: contiv-api-proxy
annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
nodeSelector:
node-role.kubernetes.io/master: "true"
tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
serviceAccountName: contiv-netmaster
containers:
- name: contiv-api-proxy
diff --git a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
index 8555c133d..3f715a473 100644
--- a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2
@@ -14,6 +14,9 @@ spec:
metadata:
labels:
k8s-app: contiv-cleanup
+ annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
priorityClassName: system-node-critical
@@ -21,8 +24,10 @@ spec:
hostNetwork: true
hostPID: true
tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
serviceAccountName: contiv-netplugin
containers:
- name: contiv-ovs-cleanup
diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
index ba17452fa..134c9c5b5 100644
--- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2
@@ -25,8 +25,10 @@ spec:
nodeSelector:
node-role.kubernetes.io/master: "true"
tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
initContainers:
- name: contiv-etcd-init
image: {{ contiv_etcd_init_image_repo }}:{{ contiv_etcd_init_image_tag }}
diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
index 5731d7c5c..55481b261 100644
--- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2
@@ -16,6 +16,7 @@ spec:
labels:
k8s-app: contiv-netmaster
annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
nodeSelector:
node-role.kubernetes.io/master: "true"
tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
serviceAccountName: contiv-netmaster
containers:
- name: contiv-netmaster
diff --git a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
index e47f711bf..4a996edea 100644
--- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2
@@ -20,6 +20,7 @@ spec:
labels:
k8s-app: contiv-netplugin
annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -28,8 +29,10 @@ spec:
hostNetwork: true
hostPID: true
tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
serviceAccountName: contiv-netplugin
initContainers:
- name: contiv-netplugin-init
diff --git a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
index 27090c62f..0ded7fe7e 100644
--- a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
+++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2
@@ -17,6 +17,7 @@ spec:
labels:
k8s-app: contiv-ovs
annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
@@ -25,8 +26,10 @@ spec:
hostNetwork: true
hostPID: true
tolerations:
- - key: node-role.kubernetes.io/master
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
containers:
# Runs ovs containers on each Kubernetes node.
- name: contiv-ovsdb-server
diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
index c872d9893..d2340eed8 100644
--- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
+++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
@@ -51,6 +51,9 @@ spec:
labels:
tier: node
k8s-app: flannel
+ annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
priorityClassName: system-node-critical
@@ -108,9 +111,10 @@ spec:
mountPath: /host/opt/cni/bin/
hostNetwork: true
tolerations:
- - key: node-role.kubernetes.io/master
- operator: Exists
- effect: NoSchedule
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
volumes:
- name: run
hostPath:
diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2
index 59740e67e..b8a9a6871 100644
--- a/roles/network_plugin/weave/templates/weave-net.yml.j2
+++ b/roles/network_plugin/weave/templates/weave-net.yml.j2
@@ -114,6 +114,9 @@ items:
metadata:
labels:
name: weave-net
+ annotations:
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
{% if kube_version|version_compare('v1.11.1', '>=') %}
priorityClassName: system-node-critical
@@ -224,8 +227,10 @@ items:
seLinuxOptions: {}
serviceAccountName: weave-net
tolerations:
- - effect: NoSchedule
- operator: Exists
+ - operator: Exists
+ # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
+ - key: CriticalAddonsOnly
+ operator: "Exists"
volumes:
- name: weavedb
hostPath:
--
GitLab