From bc6de32fafb24dcf5ea5094d12f4d4f4477a725e Mon Sep 17 00:00:00 2001
From: Holger Frydrych <frydrych@mailbox.org>
Date: Tue, 6 Aug 2019 10:37:55 +0200
Subject: [PATCH] Upgrade Cilium network plugin to v1.5.5. (#5014)
* Needs an additional cilium-operator deployment.
* Added option to enable hostPort mappings.
---
README.md | 2 +-
roles/download/defaults/main.yml | 17 +-
roles/network_plugin/cilium/defaults/main.yml | 16 +-
roles/network_plugin/cilium/tasks/main.yml | 18 +-
.../templates/000-cilium-portmap.conflist.j2 | 13 +
.../cilium/templates/cilium-config.yml.j2 | 99 ++++-
.../cilium/templates/cilium-cr.yml.j2 | 70 +++-
.../cilium/templates/cilium-crb.yml.j2 | 24 +-
.../cilium/templates/cilium-deploy.yml.j2 | 122 ++++++
.../cilium/templates/cilium-ds.yml.j2 | 392 ++++++++++--------
.../cilium/templates/cilium-sa.yml.j2 | 6 +
11 files changed, 566 insertions(+), 213 deletions(-)
create mode 100644 roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2
mode change 100755 => 100644 roles/network_plugin/cilium/templates/cilium-config.yml.j2
mode change 100755 => 100644 roles/network_plugin/cilium/templates/cilium-cr.yml.j2
create mode 100644 roles/network_plugin/cilium/templates/cilium-deploy.yml.j2
diff --git a/README.md b/README.md
index d93b82ec5..9682d3b6a 100644
--- a/README.md
+++ b/README.md
@@ -117,7 +117,7 @@ Supported Components
- [cni-plugins](https://github.com/containernetworking/plugins) v0.8.1
- [calico](https://github.com/projectcalico/calico) v3.7.3
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- - [cilium](https://github.com/cilium/cilium) v1.3.0
+ - [cilium](https://github.com/cilium/cilium) v1.5.5
- [contiv](https://github.com/contiv/install) v1.2.1
- [flanneld](https://github.com/coreos/flannel) v0.11.0
- [kube-router](https://github.com/cloudnativelabs/kube-router) v0.2.5
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 9f476357e..72561c7fd 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -73,7 +73,7 @@ cni_version: "v0.8.1"
weave_version: 2.5.2
pod_infra_version: 3.1
contiv_version: 1.2.1
-cilium_version: "v1.3.0"
+cilium_version: "v1.5.5"
kube_ovn_version: "v0.6.0"
kube_router_version: "v0.2.5"
multus_version: "v3.1.autoconf"
@@ -237,8 +237,10 @@ contiv_ovs_image_repo: "docker.io/contiv/ovs"
contiv_ovs_image_tag: "latest"
cilium_image_repo: "docker.io/cilium/cilium"
cilium_image_tag: "{{ cilium_version }}"
-cilium_init_image_repo: "docker.io/library/busybox"
-cilium_init_image_tag: "1.28.4"
+cilium_init_image_repo: "docker.io/cilium/cilium-init"
+cilium_init_image_tag: "2019-04-05"
+cilium_operator_image_repo: "docker.io/cilium/operator"
+cilium_operator_image_tag: "{{ cilium_version }}"
kube_ovn_db_image_repo: "index.alauda.cn/alaudak8s/kube-ovn-db"
kube_ovn_node_image_repo: "index.alauda.cn/alaudak8s/kube-ovn-node"
kube_ovn_cni_image_repo: "index.alauda.cn/alaudak8s/kube-ovn-cni"
@@ -415,6 +417,15 @@ downloads:
groups:
- k8s-cluster
+ cilium_operator:
+ enabled: "{{ kube_network_plugin == 'cilium' }}"
+ container: true
+ repo: "{{ cilium_operator_image_repo }}"
+ tag: "{{ cilium_operator_image_tag }}"
+ sha256: "{{ cilium_operator_digest_checksum|default(None) }}"
+ groups:
+ - k8s-cluster
+
multus:
enabled: "{{ kube_network_plugin_multus }}"
container: true
diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml
index 95cfdfcf2..9ed4b2bd4 100755
--- a/roles/network_plugin/cilium/defaults/main.yml
+++ b/roles/network_plugin/cilium/defaults/main.yml
@@ -1,7 +1,9 @@
---
# Log-level
cilium_debug: false
-cilium_disable_ipv4: false
+
+cilium_enable_ipv4: true
+cilium_enable_ipv6: false
# Etcd SSL dirs
cilium_cert_dir: /etc/cilium/certs
@@ -9,9 +11,6 @@ kube_etcd_cacert_file: ca.pem
kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
-# Cilium Network Policy directory
-cilium_policy_dir: /etc/kubernetes/policy
-
# Limits for apps
cilium_memory_limit: 500M
cilium_cpu_limit: 500m
@@ -20,3 +19,12 @@ cilium_cpu_requests: 100m
# Optional features
cilium_enable_prometheus: false
+# Enable if you want to make use of hostPort mappings
+cilium_enable_portmap: false
+
+# If upgrading from Cilium < 1.5, you may want to override some of these options
+# to prevent service disruptions. See also:
+# http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+cilium_preallocate_bpf_maps: false
+cilium_tofqdns_enable_poller: false
+cilium_enable_legacy_services: false
diff --git a/roles/network_plugin/cilium/tasks/main.yml b/roles/network_plugin/cilium/tasks/main.yml
index e830818e9..8038bdf42 100755
--- a/roles/network_plugin/cilium/tasks/main.yml
+++ b/roles/network_plugin/cilium/tasks/main.yml
@@ -34,6 +34,7 @@
- {name: cilium, file: cilium-crb.yml, type: clusterrolebinding}
- {name: cilium, file: cilium-cr.yml, type: clusterrole}
- {name: cilium, file: cilium-ds.yml, type: ds}
+ - {name: cilium, file: cilium-deploy.yml, type: deploy}
- {name: cilium, file: cilium-sa.yml, type: sa}
register: cilium_node_manifests
when:
@@ -48,7 +49,16 @@
mode: 0755
register: cni_bin_dir
-- name: Cilium | Create network policy directory
- file:
- path: "{{ cilium_policy_dir }}"
- state: directory
+- name: Cilium | Copy CNI plugins
+ unarchive:
+ src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+ dest: "/opt/cni/bin"
+ mode: 0755
+ remote_src: yes
+ when: cilium_enable_portmap
+
+- name: Cilium | Enable portmap addon
+ template:
+ src: 000-cilium-portmap.conflist.j2
+ dest: /etc/cni/net.d/000-cilium-portmap.conflist
+ when: cilium_enable_portmap
diff --git a/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2 b/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2
new file mode 100644
index 000000000..982a7c94f
--- /dev/null
+++ b/roles/network_plugin/cilium/templates/000-cilium-portmap.conflist.j2
@@ -0,0 +1,13 @@
+{
+ "cniVersion": "0.3.1",
+ "name": "cilium-portmap",
+ "plugins": [
+ {
+ "type": "cilium-cni"
+ },
+ {
+ "type": "portmap",
+ "capabilities": { "portMappings": true }
+ }
+ ]
+}
diff --git a/roles/network_plugin/cilium/templates/cilium-config.yml.j2 b/roles/network_plugin/cilium/templates/cilium-config.yml.j2
old mode 100755
new mode 100644
index cf5758465..21a803cd4
--- a/roles/network_plugin/cilium/templates/cilium-config.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-config.yml.j2
@@ -25,18 +25,76 @@ data:
key-file: "{{ cilium_cert_dir }}/key.pem"
cert-file: "{{ cilium_cert_dir }}/cert.crt"
+ # If you want metrics enabled in all of your Cilium agents, set the port for
+ # which the Cilium agents will have their metrics exposed.
+ # This option deprecates the "prometheus-serve-addr" in the
+ # "cilium-metrics-config" ConfigMap
+ # NOTE that this will open the port on ALL nodes where Cilium pods are
+ # scheduled.
+{% if cilium_enable_prometheus %}
+ prometheus-serve-addr: ":9090"
+{% endif %}
+
# If you want to run cilium in debug mode change this value to true
debug: "{{ cilium_debug }}"
- disable-ipv4: "{{ cilium_disable_ipv4 }}"
- # If you want to clean cilium state; change this value to true
+ enable-ipv4: "{{ cilium_enable_ipv4 }}"
+ enable-ipv6: "{{ cilium_enable_ipv6 }}"
+ # If a serious issue occurs during Cilium startup, this
+ # invasive option may be set to true to remove all persistent
+ # state. Endpoints will not be restored using knowledge from a
+ # prior Cilium run, so they may receive new IP addresses upon
+ # restart. This also triggers clean-cilium-bpf-state.
clean-cilium-state: "false"
- legacy-host-allows-world: "false"
+ # If you want to clean cilium BPF state, set this to true;
+ # Removes all BPF maps from the filesystem. Upon restart,
+ # endpoints are restored with the same IP addresses, however
+ # any ongoing connections may be disrupted briefly.
+ # Loadbalancing decisions will be reset, so any ongoing
+ # connections via a service may be loadbalanced to a different
+ # backend after restart.
+ clean-cilium-bpf-state: "false"
+
+ # Users who wish to specify their own custom CNI configuration file must set
+ # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration.
+ custom-cni-conf: "false"
# If you want cilium monitor to aggregate tracing for packets, set this level
# to "low", "medium", or "maximum". The higher the level, the less packets
# that will be seen in monitor output.
monitor-aggregation-level: "none"
+ # ct-global-max-entries-* specifies the maximum number of connections
+ # supported across all endpoints, split by protocol: tcp or other. One pair
+ # of maps uses these values for IPv4 connections, and another pair of maps
+ # use these values for IPv6 connections.
+ #
+ # If these values are modified, then during the next Cilium startup the
+ # tracking of ongoing connections may be disrupted. This may lead to brief
+ # policy drops or a change in loadbalancing decisions for a connection.
+ #
+ # For users upgrading from Cilium 1.2 or earlier, to minimize disruption
+ # during the upgrade process, comment out these options.
+ bpf-ct-global-tcp-max: "524288"
+ bpf-ct-global-any-max: "262144"
+
+ # Pre-allocation of map entries allows per-packet latency to be reduced, at
+ # the expense of up-front memory allocation for the entries in the maps. The
+ # default value below will minimize memory usage in the default installation;
+ # users who are sensitive to latency may consider setting this to "true".
+ #
+ # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+ # this option and behave as though it is set to "true".
+ #
+ # If this value is modified, then during the next Cilium startup the restore
+ # of existing endpoints and tracking of ongoing connections may be disrupted.
+ # This may lead to policy drops or a change in loadbalancing decisions for a
+ # connection for some time. Endpoints may need to be recreated to restore
+ # connectivity.
+ #
+ # If this option is set to "false" during an upgrade from 1.3 or earlier to
+ # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+ preallocate-bpf-maps: "{{cilium_preallocate_bpf_maps}}"
+
# Regular expression matching compatible Istio sidecar istio-proxy
# container image names
sidecar-istio-proxy-image: "cilium/istio_proxy"
@@ -47,3 +105,38 @@ data:
# - vxlan (default)
# - geneve
tunnel: "vxlan"
+
+ # Name of the cluster. Only relevant when building a mesh of clusters.
+ cluster-name: default
+
+ # Unique ID of the cluster. Must be unique across all conneted clusters and
+ # in the range of 1 and 255. Only relevant when building a mesh of clusters.
+ #cluster-id: 1
+
+ # DNS Polling periodically issues a DNS lookup for each `matchName` from
+ # cilium-agent. The result is used to regenerate endpoint policy.
+ # DNS lookups are repeated with an interval of 5 seconds, and are made for
+ # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
+ # data is used instead. An IP change will trigger a regeneration of the Cilium
+ # policy for each endpoint and increment the per cilium-agent policy
+ # repository revision.
+ #
+ # This option is disabled by default starting from version 1.4.x in favor
+ # of a more powerful DNS proxy-based implementation, see [0] for details.
+ # Enable this option if you want to use FQDN policies but do not want to use
+ # the DNS proxy.
+ #
+ # To ease upgrade, users may opt to set this option to "true".
+ # Otherwise please refer to the Upgrade Guide [1] which explains how to
+ # prepare policy rules for upgrade.
+ #
+ # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
+ # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
+ tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}"
+
+ # wait-bpf-mount makes init container wait until bpf filesystem is mounted
+ wait-bpf-mount: "false"
+
+ # Enable legacy services (prior v1.5) to prevent from terminating existing
+ # connections with services when upgrading Cilium from < v1.5 to v1.5.
+ enable-legacy-services: "{{cilium_enable_legacy_services}}"
diff --git a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2
old mode 100755
new mode 100644
index 60cdb5b6e..9bdec5aed
--- a/roles/network_plugin/cilium/templates/cilium-cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-cr.yml.j2
@@ -1,11 +1,58 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
+metadata:
+ name: cilium-operator
+rules:
+- apiGroups:
+ - ""
+ resources:
+ # to get k8s version and status
+ - componentstatuses
+ verbs:
+ - get
+- apiGroups:
+ - ""
+ resources:
+ # to automatically delete [core|kube]dns pods so that are starting to being
+ # managed by Cilium
+ - pods
+ verbs:
+ - get
+ - list
+ - watch
+ - delete
+- apiGroups:
+ - ""
+ resources:
+ # to automatically read from k8s and import the node's pod CIDR to cilium's
+ # etcd so all nodes know how to reach another pod running in in a different
+ # node.
+ - nodes
+ # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+ - services
+ - endpoints
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnetworkpolicies
+ - ciliumnetworkpolicies/status
+ - ciliumendpoints
+ - ciliumendpoints/status
+ verbs:
+ - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
metadata:
name: cilium
rules:
- apiGroups:
- - "networking.k8s.io"
+ - networking.k8s.io
resources:
- networkpolicies
verbs:
@@ -34,11 +81,16 @@ rules:
- list
- watch
- update
+ - apiGroups:
+ - ""
+ resources:
+ - nodes
+ - nodes/status
+ verbs:
+ - patch
- apiGroups:
- extensions
resources:
- - networkpolicies # FIXME remove this when we drop support for k8s NP-beta GH-1202
- - thirdpartyresources
- ingresses
verbs:
- create
@@ -46,7 +98,7 @@ rules:
- list
- watch
- apiGroups:
- - "apiextensions.k8s.io"
+ - apiextensions.k8s.io
resources:
- customresourcedefinitions
verbs:
@@ -63,12 +115,4 @@ rules:
- ciliumendpoints
- ciliumendpoints/status
verbs:
- - "*"
- - apiGroups:
- - policy
- resourceNames:
- - privileged
- resources:
- - podsecuritypolicies
- verbs:
- - use
+ - '*'
diff --git a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2
index 35994bc68..8857d31a6 100755
--- a/roles/network_plugin/cilium/templates/cilium-crb.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-crb.yml.j2
@@ -1,6 +1,19 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
+metadata:
+ name: cilium-operator
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cilium-operator
+subjects:
+- kind: ServiceAccount
+ name: cilium-operator
+ namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
metadata:
name: cilium
roleRef:
@@ -8,8 +21,9 @@ roleRef:
kind: ClusterRole
name: cilium
subjects:
- - kind: ServiceAccount
- name: cilium
- namespace: kube-system
- - kind: Group
- name: system:nodes
+- kind: ServiceAccount
+ name: cilium
+ namespace: kube-system
+- apiGroup: rbac.authorization.k8s.io
+ kind: Group
+ name: system:nodes
diff --git a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2
new file mode 100644
index 000000000..6d0e2352f
--- /dev/null
+++ b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2
@@ -0,0 +1,122 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ labels:
+ io.cilium/app: operator
+ name: cilium-operator
+ name: cilium-operator
+ namespace: kube-system
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ io.cilium/app: operator
+ name: cilium-operator
+ strategy:
+ rollingUpdate:
+ maxSurge: 1
+ maxUnavailable: 1
+ type: RollingUpdate
+ template:
+ metadata:
+ labels:
+ io.cilium/app: operator
+ name: cilium-operator
+ spec:
+ containers:
+ - args:
+ - --debug=$(CILIUM_DEBUG)
+ - --kvstore=etcd
+ - --kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config
+ command:
+ - cilium-operator
+ env:
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_DEBUG
+ valueFrom:
+ configMapKeyRef:
+ key: debug
+ name: cilium-config
+ optional: true
+ - name: CILIUM_CLUSTER_NAME
+ valueFrom:
+ configMapKeyRef:
+ key: cluster-name
+ name: cilium-config
+ optional: true
+ - name: CILIUM_CLUSTER_ID
+ valueFrom:
+ configMapKeyRef:
+ key: cluster-id
+ name: cilium-config
+ optional: true
+ - name: CILIUM_DISABLE_ENDPOINT_CRD
+ valueFrom:
+ configMapKeyRef:
+ key: disable-endpoint-crd
+ name: cilium-config
+ optional: true
+ - name: AWS_ACCESS_KEY_ID
+ valueFrom:
+ secretKeyRef:
+ key: AWS_ACCESS_KEY_ID
+ name: cilium-aws
+ optional: true
+ - name: AWS_SECRET_ACCESS_KEY
+ valueFrom:
+ secretKeyRef:
+ key: AWS_SECRET_ACCESS_KEY
+ name: cilium-aws
+ optional: true
+ - name: AWS_DEFAULT_REGION
+ valueFrom:
+ secretKeyRef:
+ key: AWS_DEFAULT_REGION
+ name: cilium-aws
+ optional: true
+ image: docker.io/cilium/operator:v1.5.5
+ imagePullPolicy: IfNotPresent
+ name: cilium-operator
+ livenessProbe:
+ httpGet:
+ path: /healthz
+ port: 9234
+ scheme: HTTP
+ initialDelaySeconds: 60
+ periodSeconds: 10
+ timeoutSeconds: 3
+ volumeMounts:
+ - mountPath: /var/lib/etcd-config
+ name: etcd-config-path
+ readOnly: true
+ - mountPath: "{{cilium_cert_dir}}"
+ name: etcd-secrets
+ readOnly: true
+ dnsPolicy: ClusterFirst
+ priorityClassName: system-node-critical
+ restartPolicy: Always
+ serviceAccount: cilium-operator
+ serviceAccountName: cilium-operator
+ volumes:
+ # To read the etcd config stored in config maps
+ - configMap:
+ defaultMode: 420
+ items:
+ - key: etcd-config
+ path: etcd.config
+ name: cilium-config
+ name: etcd-config-path
+ # To read the k8s etcd secrets in case the user might want to use TLS
+ - name: etcd-secrets
+ hostPath:
+ path: "{{cilium_cert_dir}}"
diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
index 6f62f9536..87ef2150e 100755
--- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@@ -1,219 +1,251 @@
----
apiVersion: apps/v1
kind: DaemonSet
metadata:
+ labels:
+ k8s-app: cilium
+ kubernetes.io/cluster-service: "true"
name: cilium
namespace: kube-system
spec:
- updateStrategy:
- type: "RollingUpdate"
- rollingUpdate:
- # Specifies the maximum number of Pods that can be unavailable during the update process.
- # The current default value is 1 or 100% for daemonsets; Adding an explicit value here
- # to avoid confusion, as the default value is specific to the type (daemonset/deployment).
- maxUnavailable: "100%"
selector:
matchLabels:
k8s-app: cilium
kubernetes.io/cluster-service: "true"
template:
metadata:
- labels:
- k8s-app: cilium
- kubernetes.io/cluster-service: "true"
+ annotations:
{% if cilium_enable_prometheus %}
- prometheus.io/scrape: "true"
prometheus.io/port: "9090"
+ prometheus.io/scrape: "true"
{% endif %}
+ # This annotation plus the CriticalAddonsOnly toleration makes
+ # cilium to be a critical pod in the cluster, which ensures cilium
+ # gets priority scheduling.
+ # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/
+ scheduler.alpha.kubernetes.io/critical-pod: ""
+ scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]'
+ labels:
+ k8s-app: cilium
+ kubernetes.io/cluster-service: "true"
spec:
- priorityClassName: system-node-critical
- serviceAccountName: cilium
- initContainers:
- - name: clean-cilium-state
- image: {{ cilium_init_image_repo }}:{{ cilium_init_image_tag }}
- imagePullPolicy: IfNotPresent
- command: ['sh', '-c', 'if [ "${CLEAN_CILIUM_STATE}" = "true" ]; then rm -rf /var/run/cilium/state; rm -rf /sys/fs/bpf/tc/globals/cilium_*; fi']
- volumeMounts:
- - name: bpf-maps
- mountPath: /sys/fs/bpf
- - name: cilium-run
- mountPath: /var/run/cilium
- env:
- - name: "CLEAN_CILIUM_STATE"
- valueFrom:
- configMapKeyRef:
- name: cilium-config
- optional: true
- key: clean-cilium-state
containers:
- - image: {{ cilium_image_repo }}:{{ cilium_image_tag }}
- imagePullPolicy: Always
- name: cilium-agent
- command: ["cilium-agent"]
- args:
- - "--debug=$(CILIUM_DEBUG)"
- - "--kvstore=etcd"
- - "--kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config"
- - "--disable-ipv4=$(DISABLE_IPV4)"
-{% if cilium_enable_prometheus %}
- ports:
- - name: prometheus
- containerPort: 9090
-{% endif %}
- lifecycle:
- postStart:
- exec:
- command:
- - "/cni-install.sh"
- preStop:
- exec:
- command:
- - "/cni-uninstall.sh"
- env:
- - name: "K8S_NODE_NAME"
- valueFrom:
- fieldRef:
- fieldPath: spec.nodeName
- - name: "CILIUM_DEBUG"
- valueFrom:
- configMapKeyRef:
- name: cilium-config
- key: debug
- - name: "DISABLE_IPV4"
- valueFrom:
- configMapKeyRef:
- name: cilium-config
- key: disable-ipv4
-{% if cilium_enable_prometheus %}
- # Note: this variable is a no-op if not defined, and is used in the
- # prometheus examples.
- - name: "CILIUM_PROMETHEUS_SERVE_ADDR"
- valueFrom:
- configMapKeyRef:
- name: cilium-metrics-config
- optional: true
- key: prometheus-serve-addr
-{% endif %}
- - name: "CILIUM_LEGACY_HOST_ALLOWS_WORLD"
- valueFrom:
- configMapKeyRef:
- name: cilium-config
- optional: true
- key: legacy-host-allows-world
- - name: "CILIUM_SIDECAR_ISTIO_PROXY_IMAGE"
- valueFrom:
- configMapKeyRef:
- name: cilium-config
- key: sidecar-istio-proxy-image
- optional: true
- - name: "CILIUM_TUNNEL"
- valueFrom:
- configMapKeyRef:
- key: tunnel
- name: cilium-config
- optional: true
- - name: "CILIUM_MONITOR_AGGREGATION_LEVEL"
- valueFrom:
- configMapKeyRef:
- key: monitor-aggregation-level
- name: cilium-config
- optional: true
- resources:
- limits:
- cpu: {{ cilium_cpu_limit }}
- memory: {{ cilium_memory_limit }}
- requests:
- cpu: {{ cilium_cpu_requests }}
- memory: {{ cilium_memory_requests }}
- livenessProbe:
+ - args:
+ - --kvstore=etcd
+ - --kvstore-opt=etcd.config=/var/lib/etcd-config/etcd.config
+ - --config-dir=/tmp/cilium/config-map
+ command:
+ - cilium-agent
+ env:
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_K8S_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ - name: CILIUM_CLUSTERMESH_CONFIG
+ value: /var/lib/cilium/clustermesh/
+ image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+ imagePullPolicy: IfNotPresent
+ resources:
+ limits:
+ cpu: {{ cilium_cpu_limit }}
+ memory: {{ cilium_memory_limit }}
+ requests:
+ cpu: {{ cilium_cpu_requests }}
+ memory: {{ cilium_memory_requests }}
+ lifecycle:
+ postStart:
exec:
command:
- - cilium
- - status
- initialDelaySeconds: 15
- failureThreshold: 10
- periodSeconds: 10
- readinessProbe:
+ - /cni-install.sh
+ preStop:
exec:
command:
- - cilium
- - status
- initialDelaySeconds: 5
- periodSeconds: 5
- volumeMounts:
- - name: bpf-maps
- mountPath: /sys/fs/bpf
- - name: cilium-run
- mountPath: /var/run/cilium
- - name: cni-path
- mountPath: /host/opt/cni/bin
- - name: etc-cni-netd
- mountPath: /host/etc/cni/net.d
+ - /cni-uninstall.sh
+ livenessProbe:
+ exec:
+ command:
+ - cilium
+ - status
+ - --brief
+ failureThreshold: 10
+ # The initial delay for the liveness probe is intentionally large to
+ # avoid an endless kill & restart cycle if in the event that the initial
+ # bootstrapping takes longer than expected.
+ initialDelaySeconds: 120
+ periodSeconds: 30
+ successThreshold: 1
+ timeoutSeconds: 5
+ name: cilium-agent
+{% if cilium_enable_prometheus %}
+ ports:
+ - containerPort: 9090
+ hostPort: 9090
+ name: prometheus
+ protocol: TCP
+{% endif %}
+ readinessProbe:
+ exec:
+ command:
+ - cilium
+ - status
+ - --brief
+ failureThreshold: 3
+ initialDelaySeconds: 5
+ periodSeconds: 30
+ successThreshold: 1
+ timeoutSeconds: 5
+ securityContext:
+ capabilities:
+ add:
+ - NET_ADMIN
+ - SYS_MODULE
+ privileged: true
+ volumeMounts:
+ - mountPath: /sys/fs/bpf
+ name: bpf-maps
+ - mountPath: /var/run/cilium
+ name: cilium-run
+ - mountPath: /host/opt/cni/bin
+ name: cni-path
+ - mountPath: /host/etc/cni/net.d
+ name: etc-cni-netd
{% if container_manager == 'docker' %}
- - name: docker-socket
- mountPath: /var/run/docker.sock
- readOnly: true
+ - mountPath: /var/run/docker.sock
+ name: docker-socket
+ readOnly: true
{% else %}
- name: "{{ container_manager }}-socket"
mountPath: {{ cri_socket }}
readOnly: true
{% endif %}
- - name: etcd-config-path
- mountPath: /var/lib/etcd-config
- readOnly: true
- - name: cilium-certs
- mountPath: {{ cilium_cert_dir }}
- readOnly: true
- securityContext:
- capabilities:
- add:
- - "NET_ADMIN"
- privileged: true
- hostNetwork: true
+ - mountPath: /var/lib/etcd-config
+ name: etcd-config-path
+ readOnly: true
+ - mountPath: "{{cilium_cert_dir}}"
+ name: etcd-secrets
+ readOnly: true
+ - mountPath: /var/lib/cilium/clustermesh
+ name: clustermesh-secrets
+ readOnly: true
+ - mountPath: /tmp/cilium/config-map
+ name: cilium-config-path
+ readOnly: true
+ # Needed to be able to load kernel modules
+ - mountPath: /lib/modules
+ name: lib-modules
+ readOnly: true
dnsPolicy: ClusterFirstWithHostNet
+ hostNetwork: true
+ hostPID: false
+ initContainers:
+ - command:
+ - /init-container.sh
+ env:
+ - name: CLEAN_CILIUM_STATE
+ valueFrom:
+ configMapKeyRef:
+ key: clean-cilium-state
+ name: cilium-config
+ optional: true
+ - name: CLEAN_CILIUM_BPF_STATE
+ valueFrom:
+ configMapKeyRef:
+ key: clean-cilium-bpf-state
+ name: cilium-config
+ optional: true
+ - name: CILIUM_WAIT_BPF_MOUNT
+ valueFrom:
+ configMapKeyRef:
+ key: wait-bpf-mount
+ name: cilium-config
+ optional: true
+ image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
+ imagePullPolicy: IfNotPresent
+ name: clean-cilium-state
+ securityContext:
+ capabilities:
+ add:
+ - NET_ADMIN
+ privileged: true
+ volumeMounts:
+ - mountPath: /sys/fs/bpf
+ name: bpf-maps
+ - mountPath: /var/run/cilium
+ name: cilium-run
+ priorityClassName: system-node-critical
+ restartPolicy: Always
+ serviceAccount: cilium
+ serviceAccountName: cilium
+ terminationGracePeriodSeconds: 1
+ tolerations:
+ - operator: Exists
volumes:
# To keep state between restarts / upgrades
- - name: cilium-run
- hostPath:
- path: /var/run/cilium
- # To keep state between restarts / upgrades
- - name: bpf-maps
- hostPath:
- path: /sys/fs/bpf
+ - hostPath:
+ path: /var/run/cilium
+ type: DirectoryOrCreate
+ name: cilium-run
+ # To keep state between restarts / upgrades for bpf maps
+ - hostPath:
+ path: /sys/fs/bpf
+ type: DirectoryOrCreate
+ name: bpf-maps
{% if container_manager == 'docker' %}
# To read docker events from the node
- - name: docker-socket
- hostPath:
- path: /var/run/docker.sock
+ - hostPath:
+ path: /var/run/docker.sock
+ type: Socket
+ name: docker-socket
{% else %}
# To read crio events from the node
- - name: {{ container_manager }}-socket
- hostPath:
- path: {{ cri_socket }}
+ - hostPath:
+ path: {{ cri_socket }}
+ type: Socket
+ name: {{ container_manager }}-socket
{% endif %}
# To install cilium cni plugin in the host
- - name: cni-path
- hostPath:
- path: /opt/cni/bin
+ - hostPath:
+ path: /opt/cni/bin
+ type: DirectoryOrCreate
+ name: cni-path
# To install cilium cni configuration in the host
- - name: etc-cni-netd
- hostPath:
- path: /etc/cni/net.d
+ - hostPath:
+ path: /etc/cni/net.d
+ type: DirectoryOrCreate
+ name: etc-cni-netd
+ # To be able to load kernel modules
+ - hostPath:
+ path: /lib/modules
+ name: lib-modules
# To read the etcd config stored in config maps
- - name: etcd-config-path
- configMap:
- name: cilium-config
- items:
- - key: etcd-config
- path: etcd.config
+ - configMap:
+ defaultMode: 420
+ items:
+ - key: etcd-config
+ path: etcd.config
+ name: cilium-config
+ name: etcd-config-path
# To read the k8s etcd secrets in case the user might want to use TLS
- - name: cilium-certs
- hostPath:
- path: {{ cilium_cert_dir }}
-
- restartPolicy: Always
- tolerations:
- - operator: Exists
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- - key: CriticalAddonsOnly
- operator: "Exists"
+ - name: etcd-secrets
+ hostPath:
+ path: "{{cilium_cert_dir}}"
+ # To read the clustermesh configuration
+ - name: clustermesh-secrets
+ secret:
+ defaultMode: 420
+ optional: true
+ secretName: cilium-clustermesh
+ # To read the configuration from the config map
+ - configMap:
+ name: cilium-config
+ name: cilium-config-path
+ updateStrategy:
+ rollingUpdate:
+ # Specifies the maximum number of Pods that can be unavailable during the update process.
+ maxUnavailable: 2
+ type: RollingUpdate
diff --git a/roles/network_plugin/cilium/templates/cilium-sa.yml.j2 b/roles/network_plugin/cilium/templates/cilium-sa.yml.j2
index c03ac59b4..2c5c0cb19 100755
--- a/roles/network_plugin/cilium/templates/cilium-sa.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-sa.yml.j2
@@ -1,6 +1,12 @@
---
apiVersion: v1
kind: ServiceAccount
+metadata:
+ name: cilium-operator
+ namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
metadata:
name: cilium
namespace: kube-system
--
GitLab