From 7da3dbcb397f08986f6796e5eed547ff2548344a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Necatican=20Y=C4=B1ld=C4=B1r=C4=B1m?=
<contact@necatican.com>
Date: Mon, 19 Sep 2022 12:14:31 +0300
Subject: [PATCH] Cilium 1.12 Upgrade (#9225)
* Drop support for Cilium < 1.10
Signed-off-by: necatican <necaticanyildirim@gmail.com>
* Synchronize Cilium templates for 1.11.7
Signed-off-by: necatican <contact@necatican.com>
* Set Cilium v1.12.1 as the default version
Signed-off-by: necatican <contact@necatican.com>
Signed-off-by: necatican <necaticanyildirim@gmail.com>
Signed-off-by: necatican <contact@necatican.com>
---
README.md | 2 +-
docs/cilium.md | 2 +-
.../group_vars/k8s_cluster/k8s-net-cilium.yml | 3 +-
roles/download/defaults/main.yml | 2 +-
roles/network_plugin/cilium/defaults/main.yml | 11 +-
roles/network_plugin/cilium/tasks/check.yml | 9 +-
.../templates/cilium-operator/cr.yml.j2 | 74 ++--
.../templates/cilium-operator/deploy.yml.j2 | 117 +++---
.../cilium/templates/cilium/config.yml.j2 | 46 +--
.../cilium/templates/cilium/cr.yml.j2 | 58 ++-
.../cilium/templates/cilium/crb.yml.j2 | 5 -
.../cilium/templates/cilium/ds.yml.j2 | 332 +++++++++---------
12 files changed, 289 insertions(+), 372 deletions(-)
diff --git a/README.md b/README.md
index 44403da09..ae3e5b36d 100644
--- a/README.md
+++ b/README.md
@@ -144,7 +144,7 @@ Note: Upstart/SysV init based OS types are not supported.
- [cni-plugins](https://github.com/containernetworking/plugins) v1.1.1
- [calico](https://github.com/projectcalico/calico) v3.23.3
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- - [cilium](https://github.com/cilium/cilium) v1.11.7
+ - [cilium](https://github.com/cilium/cilium) v1.12.1
- [flannel](https://github.com/flannel-io/flannel) v0.18.1
- [kube-ovn](https://github.com/alauda/kube-ovn) v1.9.7
- [kube-router](https://github.com/cloudnativelabs/kube-router) v1.5.1
diff --git a/docs/cilium.md b/docs/cilium.md
index e9c3e0d2b..e907d53cd 100644
--- a/docs/cilium.md
+++ b/docs/cilium.md
@@ -56,7 +56,7 @@ cilium_operator_extra_volume_mounts:
## Choose Cilium version
```yml
-cilium_version: v1.11.3
+cilium_version: v1.12.1
```
## Add variable to config
diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml
index e82b76138..d6e5bfa61 100644
--- a/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-cilium.yml
@@ -1,5 +1,5 @@
---
-# cilium_version: "v1.11.7"
+# cilium_version: "v1.12.1"
# Log-level
# cilium_debug: false
@@ -118,6 +118,7 @@
# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
# cilium_ip_masq_agent_enable: false
+
### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
# cilium_non_masquerade_cidrs:
# - 10.0.0.0/8
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 1889dfbf1..96595d325 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -115,7 +115,7 @@ flannel_cni_version: "v1.1.0"
cni_version: "v1.1.1"
weave_version: 2.8.1
pod_infra_version: "3.6"
-cilium_version: "v1.11.7"
+cilium_version: "v1.12.1"
kube_ovn_version: "v1.9.7"
kube_ovn_dpdk_version: "19.11-{{ kube_ovn_version }}"
kube_router_version: "v1.5.1"
diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml
index c590637dc..0e624e53c 100644
--- a/roles/network_plugin/cilium/defaults/main.yml
+++ b/roles/network_plugin/cilium/defaults/main.yml
@@ -1,4 +1,5 @@
---
+cilium_min_version_required: "1.10"
# Log-level
cilium_debug: false
@@ -7,7 +8,7 @@ cilium_enable_ipv4: true
cilium_enable_ipv6: false
# Cilium agent health port
-cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
+cilium_agent_health_port: "{%- if cilium_version | regex_replace('v') is version('1.11.6', '>=') -%}9879 {%- else -%} 9876 {%- endif -%}"
# Identity allocation mode selects how identities are shared between cilium
# nodes by setting how they are stored. The options are "crd" or "kvstore".
@@ -106,6 +107,7 @@ cilium_wireguard_userspace_fallback: false
# https://docs.cilium.io/en/stable/concepts/networking/masquerading/
# By default, all packets from a pod destined to an IP address outside of the cilium_native_routing_cidr range are masqueraded
cilium_ip_masq_agent_enable: false
+
### A packet sent from a pod to a destination which belongs to any CIDR from the nonMasqueradeCIDRs is not going to be masqueraded
cilium_non_masquerade_cidrs:
- 10.0.0.0/8
@@ -201,7 +203,7 @@ cilium_cgroup_host_root: "/run/cilium/cgroupv2"
# Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
# sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
-cilium_bpf_map_dynamic_size_ratio: "{%- if cilium_version | regex_replace('v') is version('1.8', '>=') -%}0.0025{%- else -%}0.0{%- endif -%}"
+cilium_bpf_map_dynamic_size_ratio: "0.0025"
# -- Enables masquerading of IPv4 traffic leaving the node from endpoints.
# Available for Cilium v1.10 and up
@@ -240,3 +242,8 @@ cilium_disable_cnp_status_updates: true
# Configure how long to wait for the Cilium DaemonSet to be ready again
cilium_rolling_restart_wait_retries_count: 30
cilium_rolling_restart_wait_retries_delay_seconds: 10
+
+# Cilium changed the default metrics exporter ports in 1.12
+cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}"
+cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}"
+cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}"
diff --git a/roles/network_plugin/cilium/tasks/check.yml b/roles/network_plugin/cilium/tasks/check.yml
index fffa1b53a..c65591f66 100644
--- a/roles/network_plugin/cilium/tasks/check.yml
+++ b/roles/network_plugin/cilium/tasks/check.yml
@@ -48,13 +48,10 @@
msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
when: cilium_encryption_enabled
-- name: Stop if `cilium_encryption_type` is set to "wireguard" and cilium_version is < v1.10.0
+- name: Stop if cilium_version is < v1.10.0
assert:
- that: cilium_version | regex_replace('v') is version('1.10', '>')
- msg: "cilium_encryption_type is set to 'wireguard' but cilium_version is < v1.10.0"
- when:
- - cilium_encryption_enabled
- - cilium_encryption_type == "wireguard"
+ that: cilium_version | regex_replace('v') is version(cilium_min_version_required, '>=')
+ msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"
# TODO: Clean this task up when we drop backward compatibility support for `cilium_ipsec_enabled`
- name: Set `cilium_encryption_type` to "ipsec" and if `cilium_ipsec_enabled` is true
diff --git a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
index 9f7a71174..8a40a6641 100644
--- a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
@@ -39,7 +39,14 @@ rules:
- get
- list
- watch
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
+- apiGroups:
+ - ""
+ resources:
+ - services
+ verbs:
+ - get
+ - list
+ - watch
- apiGroups:
- ""
resources:
@@ -47,22 +54,14 @@ rules:
- services/status
verbs:
- update
-{% endif %}
- apiGroups:
- ""
resources:
- # to automatically read from k8s and import the node's pod CIDR to cilium's
- # etcd so all nodes know how to reach another pod running in in a different
- # node.
- - nodes
# to perform the translation of a CNP that contains `ToGroup` to its endpoints
- services
- endpoints
# to check apiserver connectivity
- namespaces
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- - componentstatuses
-{% endif %}
verbs:
- get
- list
@@ -72,26 +71,22 @@ rules:
resources:
- ciliumnetworkpolicies
- ciliumnetworkpolicies/status
+ - ciliumnetworkpolicies/finalizers
- ciliumclusterwidenetworkpolicies
- ciliumclusterwidenetworkpolicies/status
+ - ciliumclusterwidenetworkpolicies/finalizers
- ciliumendpoints
- ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
+ - ciliumendpoints/finalizers
- ciliumnodes
- ciliumnodes/status
+ - ciliumnodes/finalizers
- ciliumidentities
- ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
- - ciliumnetworkpolicies/finalizers
- - ciliumclusterwidenetworkpolicies/finalizers
- - ciliumendpoints/finalizers
- - ciliumnodes/finalizers
- ciliumidentities/finalizers
- ciliumlocalredirectpolicies
- ciliumlocalredirectpolicies/status
- ciliumlocalredirectpolicies/finalizers
-{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
- ciliumendpointslices
{% endif %}
@@ -101,12 +96,7 @@ rules:
- ciliumenvoyconfigs
{% endif %}
verbs:
- - get
- - list
- - watch
- - create
- - update
- - delete
+ - '*'
- apiGroups:
- apiextensions.k8s.io
resources:
@@ -117,16 +107,12 @@ rules:
- list
- update
- watch
-{% if cilium_version | regex_replace('v') is version('1.8', '>=') %}
- # For cilium-operator running in HA mode.
- #
- # Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
- # between mulitple running instances.
- # The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
- # common and fewer objects in the cluster watch "all Leases".
- # The support for leases was introduced in coordination.k8s.io/v1 during Kubernetes 1.14 release.
- # In Cilium we currently don't support HA mode for K8s version < 1.14. This condition make sure
- # that we only authorize access to leases resources in supported K8s versions.
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between multiple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
- apiGroups:
- coordination.k8s.io
resources:
@@ -135,4 +121,26 @@ rules:
- create
- get
- update
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+- apiGroups:
+ - apiextensions.k8s.io
+ resources:
+ - customresourcedefinitions
+ verbs:
+ - update
+ resourceNames:
+ - ciliumbgploadbalancerippools.cilium.io
+ - ciliumbgppeeringpolicies.cilium.io
+ - ciliumclusterwideenvoyconfigs.cilium.io
+ - ciliumclusterwidenetworkpolicies.cilium.io
+ - ciliumegressgatewaypolicies.cilium.io
+ - ciliumegressnatpolicies.cilium.io
+ - ciliumendpoints.cilium.io
+ - ciliumendpointslices.cilium.io
+ - ciliumenvoyconfigs.cilium.io
+ - ciliumexternalworkloads.cilium.io
+ - ciliumidentities.cilium.io
+ - ciliumlocalredirectpolicies.cilium.io
+ - ciliumnetworkpolicies.cilium.io
+ - ciliumnodes.cilium.io
{% endif %}
diff --git a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
index ab8a31926..5a5bd4a92 100644
--- a/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-operator/deploy.yml.j2
@@ -2,11 +2,11 @@
apiVersion: apps/v1
kind: Deployment
metadata:
+ name: cilium-operator
+ namespace: kube-system
labels:
io.cilium/app: operator
name: cilium-operator
- name: cilium-operator
- namespace: kube-system
spec:
replicas: {{ cilium_operator_replicas }}
selector:
@@ -22,29 +22,22 @@ spec:
metadata:
{% if cilium_enable_prometheus %}
annotations:
- prometheus.io/port: "6942"
+ prometheus.io/port: "{{ cilium_operator_scrape_port }}"
prometheus.io/scrape: "true"
{% endif %}
labels:
io.cilium/app: operator
name: cilium-operator
spec:
- # In HA mode, cilium-operator pods must not be scheduled on the same
- # node as they will clash with each other.
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: io.cilium/app
- operator: In
- values:
- - operator
- topologyKey: "kubernetes.io/hostname"
containers:
- - args:
- - --debug=$(CILIUM_DEBUG)
+ - name: cilium-operator
+ image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ command:
+ - cilium-operator
+ args:
- --config-dir=/tmp/cilium/config-map
+ - --debug=$(CILIUM_DEBUG)
{% if cilium_operator_custom_args is string %}
- {{ cilium_operator_custom_args }}
{% else %}
@@ -52,14 +45,7 @@ spec:
- {{ flag }}
{% endfor %}
{% endif %}
- command:
- - cilium-operator
env:
- - name: POD_NAMESPACE
- valueFrom:
- fieldRef:
- apiVersion: v1
- fieldPath: metadata.namespace
- name: K8S_NODE_NAME
valueFrom:
fieldRef:
@@ -76,45 +62,23 @@ spec:
key: debug
name: cilium-config
optional: true
-# We are already mounting the whole ConfigMap as a directory.
-# https://github.com/cilium/cilium/pull/10347
-{% if cilium_version | regex_replace('v') is version('1.8', '<') %}
- - name: CILIUM_CLUSTER_NAME
- valueFrom:
- configMapKeyRef:
- key: cluster-name
- name: cilium-config
- optional: true
- - name: CILIUM_CLUSTER_ID
- valueFrom:
- configMapKeyRef:
- key: cluster-id
- name: cilium-config
- optional: true
- - name: CILIUM_DISABLE_ENDPOINT_CRD
- valueFrom:
- configMapKeyRef:
- key: disable-endpoint-crd
- name: cilium-config
- optional: true
-{% endif %}
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
- key: AWS_ACCESS_KEY_ID
name: cilium-aws
+ key: AWS_ACCESS_KEY_ID
optional: true
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
- key: AWS_SECRET_ACCESS_KEY
name: cilium-aws
+ key: AWS_SECRET_ACCESS_KEY
optional: true
- name: AWS_DEFAULT_REGION
valueFrom:
secretKeyRef:
- key: AWS_DEFAULT_REGION
name: cilium-aws
+ key: AWS_DEFAULT_REGION
optional: true
{% if cilium_kube_proxy_replacement == 'strict' %}
- name: KUBERNETES_SERVICE_HOST
@@ -122,14 +86,16 @@ spec:
- name: KUBERNETES_SERVICE_PORT
value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
{% endif %}
- image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
- imagePullPolicy: {{ k8s_image_pull_policy }}
- name: cilium-operator
{% if cilium_enable_prometheus %}
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
ports:
- - containerPort: 6942
- hostPort: 6942
- name: prometheus
+ - name: prometheus
+ containerPort: {{ cilium_operator_scrape_port }}
+ hostPort: {{ cilium_operator_scrape_port }}
protocol: TCP
{% endif %}
livenessProbe:
@@ -146,46 +112,55 @@ spec:
periodSeconds: 10
timeoutSeconds: 3
volumeMounts:
+ - name: cilium-config-path
+ mountPath: /tmp/cilium/config-map
+ readOnly: true
{% if cilium_identity_allocation_mode == "kvstore" %}
- - mountPath: /var/lib/etcd-config
- name: etcd-config-path
+ - name: etcd-config-path
+ mountPath: /var/lib/etcd-config
readOnly: true
- - mountPath: "{{cilium_cert_dir}}"
- name: etcd-secrets
+ - name: etcd-secrets
+ mountPath: "{{cilium_cert_dir}}"
readOnly: true
{% endif %}
- - mountPath: /tmp/cilium/config-map
- name: cilium-config-path
- readOnly: true
{% for volume_mount in cilium_operator_extra_volume_mounts %}
- {{ volume_mount | to_nice_yaml(indent=2) | indent(14) }}
{% endfor %}
- dnsPolicy: ClusterFirst
- priorityClassName: system-node-critical
+ hostNetwork: true
+ dnsPolicy: ClusterFirstWithHostNet
restartPolicy: Always
+ priorityClassName: system-node-critical
serviceAccount: cilium-operator
serviceAccountName: cilium-operator
- hostNetwork: true
+ # In HA mode, cilium-operator pods must not be scheduled on the same
+ # node as they will clash with each other.
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - topologyKey: kubernetes.io/hostname
+ labelSelector:
+ matchLabels:
+ io.cilium/app: operator
tolerations:
- operator: Exists
volumes:
+ - name: cilium-config-path
+ configMap:
+ name: cilium-config
{% if cilium_identity_allocation_mode == "kvstore" %}
# To read the etcd config stored in config maps
- - configMap:
+ - name: etcd-config-path
+ configMap:
+ name: cilium-config
defaultMode: 420
items:
- key: etcd-config
path: etcd.config
- name: cilium-config
- name: etcd-config-path
# To read the k8s etcd secrets in case the user might want to use TLS
- name: etcd-secrets
hostPath:
path: "{{cilium_cert_dir}}"
{% endif %}
- - configMap:
- name: cilium-config
- name: cilium-config-path
{% for volume in cilium_operator_extra_volumes %}
- {{ volume | to_nice_yaml(indent=2) | indent(10) }}
{% endfor %}
diff --git a/roles/network_plugin/cilium/templates/cilium/config.yml.j2 b/roles/network_plugin/cilium/templates/cilium/config.yml.j2
index 75232991f..fadc87b64 100644
--- a/roles/network_plugin/cilium/templates/cilium/config.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/config.yml.j2
@@ -41,8 +41,8 @@ data:
# NOTE that this will open the port on ALL nodes where Cilium pods are
# scheduled.
{% if cilium_enable_prometheus %}
- prometheus-serve-addr: ":9090"
- operator-prometheus-serve-addr: ":6942"
+ prometheus-serve-addr: ":{{ cilium_agent_scrape_port }}"
+ operator-prometheus-serve-addr: ":{{ cilium_operator_scrape_port }}"
enable-metrics: "true"
{% endif %}
@@ -127,31 +127,6 @@ data:
cluster-id: "{{ cilium_cluster_id }}"
{% endif %}
-# `tofqdns-enable-poller` is deprecated in 1.8, removed in 1.9
-# https://github.com/cilium/cilium/issues/8604
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
- # DNS Polling periodically issues a DNS lookup for each `matchName` from
- # cilium-agent. The result is used to regenerate endpoint policy.
- # DNS lookups are repeated with an interval of 5 seconds, and are made for
- # A(IPv4) and AAAA(IPv6) addresses. Should a lookup fail, the most recent IP
- # data is used instead. An IP change will trigger a regeneration of the Cilium
- # policy for each endpoint and increment the per cilium-agent policy
- # repository revision.
- #
- # This option is disabled by default starting from version 1.4.x in favor
- # of a more powerful DNS proxy-based implementation, see [0] for details.
- # Enable this option if you want to use FQDN policies but do not want to use
- # the DNS proxy.
- #
- # To ease upgrade, users may opt to set this option to "true".
- # Otherwise please refer to the Upgrade Guide [1] which explains how to
- # prepare policy rules for upgrade.
- #
- # [0] http://docs.cilium.io/en/stable/policy/language/#dns-based
- # [1] http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
- tofqdns-enable-poller: "{{cilium_tofqdns_enable_poller}}"
-{% endif %}
-
# `wait-bpf-mount` is removed after v1.10.4
# https://github.com/cilium/cilium/commit/d2217045cb3726a7f823174e086913b69b8090da
{% if cilium_version | regex_replace('v') is version('1.10.4', '<') %}
@@ -159,14 +134,6 @@ data:
wait-bpf-mount: "false"
{% endif %}
-# `enable-legacy-services` is deprecated in 1.6, removed in 1.9
-# https://github.com/cilium/cilium/pull/10255
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
- # Enable legacy services (prior v1.5) to prevent from terminating existing
- # connections with services when upgrading Cilium from < v1.5 to v1.5.
- enable-legacy-services: "{{cilium_enable_legacy_services}}"
-{% endif %}
-
kube-proxy-replacement: "{{ cilium_kube_proxy_replacement }}"
# `native-routing-cidr` is deprecated in 1.10, removed in 1.12.
@@ -191,7 +158,7 @@ data:
{% if cilium_enable_hubble %}
enable-hubble: "true"
{% if cilium_enable_hubble_metrics %}
- hubble-metrics-server: ":9091"
+ hubble-metrics-server: ":{{ cilium_hubble_scrape_port }}"
hubble-metrics:
{% for hubble_metrics_cycle in cilium_hubble_metrics %}
{{ hubble_metrics_cycle }}
@@ -228,9 +195,7 @@ data:
{% endif %}
# IPAM settings
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
ipam: "{{ cilium_ipam_mode }}"
-{% endif %}
agent-health-port: "{{ cilium_agent_health_port }}"
@@ -240,10 +205,8 @@ data:
bpf-map-dynamic-size-ratio: "{{ cilium_bpf_map_dynamic_size_ratio }}"
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
enable-ipv4-masquerade: "{{ cilium_enable_ipv4_masquerade }}"
enable-ipv6-masquerade: "{{ cilium_enable_ipv6_masquerade }}"
-{% endif %}
enable-bpf-masquerade: "{{ cilium_enable_bpf_masquerade }}"
@@ -258,7 +221,6 @@ data:
enable-bpf-clock-probe: "{{ cilium_enable_bpf_clock_probe }}"
disable-cnp-status-updates: "{{ cilium_disable_cnp_status_updates }}"
-
{% if cilium_ip_masq_agent_enable %}
---
apiVersion: v1
@@ -274,4 +236,4 @@ data:
{% endfor %}
masqLinkLocal: {{ cilium_masq_link_local|bool }}
resyncInterval: "{{ cilium_ip_masq_resync_interval }}"
-{% endif %}
+{% endif %}
\ No newline at end of file
diff --git a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
index 02d8e6bf7..a16211c17 100644
--- a/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/cr.yml.j2
@@ -7,9 +7,6 @@ rules:
- apiGroups:
- networking.k8s.io
resources:
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- - ingresses
-{% endif %}
- networkpolicies
verbs:
- get
@@ -28,34 +25,25 @@ rules:
resources:
- namespaces
- services
- - nodes
+ - pods
- endpoints
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
- - componentstatuses
-{% endif %}
+ - nodes
verbs:
- get
- list
- watch
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
+{% if cilium_version | regex_replace('v') is version('1.12', '<') %}
- apiGroups:
- - extensions
+ - ""
resources:
- - ingresses
+ - pods
+ - pods/finalizers
verbs:
- - create
- get
- list
- watch
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.7', '>') %}
-- apiGroups:
- - ""
- resources:
- - pods/finalizers
- verbs:
- update
-{% endif %}
+ - delete
- apiGroups:
- ""
resources:
@@ -66,6 +54,7 @@ rules:
- list
- watch
- update
+{% endif %}
- apiGroups:
- ""
resources:
@@ -78,47 +67,45 @@ rules:
resources:
- customresourcedefinitions
verbs:
+ # Deprecated for removal in v1.10
- create
- - get
- list
- watch
- update
+
+ # This is used when validating policies in preflight. This will need to stay
+ # until we figure out how to avoid "get" inside the preflight, and then
+ # should be removed ideally.
+ - get
- apiGroups:
- cilium.io
resources:
- ciliumnetworkpolicies
- ciliumnetworkpolicies/status
-{% if cilium_version | regex_replace('v') is version('1.7', '>=') %}
- ciliumclusterwidenetworkpolicies
- ciliumclusterwidenetworkpolicies/status
-{% endif %}
- ciliumendpoints
- ciliumendpoints/status
-{% if cilium_version | regex_replace('v') is version('1.6', '>=') %}
- ciliumnodes
- ciliumnodes/status
- ciliumidentities
- - ciliumidentities/status
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '>=') %}
- - ciliumnetworkpolicies/finalizers
- - ciliumclusterwidenetworkpolicies/finalizers
- - ciliumendpoints/finalizers
- - ciliumnodes/finalizers
- - ciliumidentities/finalizers
- ciliumlocalredirectpolicies
- ciliumlocalredirectpolicies/status
- - ciliumlocalredirectpolicies/finalizers
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
- ciliumegressnatpolicies
-{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
- ciliumendpointslices
{% endif %}
{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
- ciliumbgploadbalancerippools
- ciliumbgppeeringpolicies
+{% endif %}
+{% if cilium_version | regex_replace('v') is version('1.11.5', '<') %}
+ - ciliumnetworkpolicies/finalizers
+ - ciliumclusterwidenetworkpolicies/finalizers
+ - ciliumendpoints/finalizers
+ - ciliumnodes/finalizers
+ - ciliumidentities/finalizers
+ - ciliumlocalredirectpolicies/finalizers
{% endif %}
verbs:
- '*'
@@ -128,6 +115,7 @@ rules:
resources:
- ciliumclusterwideenvoyconfigs
- ciliumenvoyconfigs
+ - ciliumegressgatewaypolicies
verbs:
- list
- watch
diff --git a/roles/network_plugin/cilium/templates/cilium/crb.yml.j2 b/roles/network_plugin/cilium/templates/cilium/crb.yml.j2
index f7516d707..d23897fa0 100644
--- a/roles/network_plugin/cilium/templates/cilium/crb.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/crb.yml.j2
@@ -11,8 +11,3 @@ subjects:
- kind: ServiceAccount
name: cilium
namespace: kube-system
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
-- apiGroup: rbac.authorization.k8s.io
- kind: Group
- name: system:nodes
-{% endif %}
diff --git a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2
index d5ceaa676..2ce442062 100644
--- a/roles/network_plugin/cilium/templates/cilium/ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium/ds.yml.j2
@@ -2,37 +2,37 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
- labels:
- k8s-app: cilium
name: cilium
namespace: kube-system
+ labels:
+ k8s-app: cilium
spec:
selector:
matchLabels:
k8s-app: cilium
+ updateStrategy:
+ rollingUpdate:
+ # Specifies the maximum number of Pods that can be unavailable during the update process.
+ maxUnavailable: 2
+ type: RollingUpdate
template:
metadata:
annotations:
{% if cilium_enable_prometheus %}
- prometheus.io/port: "9090"
+ prometheus.io/port: "{{ cilium_agent_scrape_port }}"
prometheus.io/scrape: "true"
{% endif %}
scheduler.alpha.kubernetes.io/tolerations: '[{"key":"dedicated","operator":"Equal","value":"master","effect":"NoSchedule"}]'
labels:
k8s-app: cilium
spec:
- affinity:
- podAntiAffinity:
- requiredDuringSchedulingIgnoredDuringExecution:
- - labelSelector:
- matchExpressions:
- - key: k8s-app
- operator: In
- values:
- - cilium
- topologyKey: kubernetes.io/hostname
containers:
- - args:
+ - name: cilium-agent
+ image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ command:
+ - cilium-agent
+ args:
- --config-dir=/tmp/cilium/config-map
{% if cilium_mtu != "" %}
- --mtu={{ cilium_mtu }}
@@ -44,54 +44,6 @@ spec:
- {{ flag }}
{% endfor %}
{% endif %}
- command:
- - cilium-agent
- env:
- - name: K8S_NODE_NAME
- valueFrom:
- fieldRef:
- apiVersion: v1
- fieldPath: spec.nodeName
- - name: CILIUM_K8S_NAMESPACE
- valueFrom:
- fieldRef:
- apiVersion: v1
- fieldPath: metadata.namespace
- - name: CILIUM_CLUSTERMESH_CONFIG
- value: /var/lib/cilium/clustermesh/
-{% if cilium_kube_proxy_replacement == 'strict' %}
- - name: KUBERNETES_SERVICE_HOST
- value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
- - name: KUBERNETES_SERVICE_PORT
- value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
-{% endif %}
-{% for env_var in cilium_agent_extra_env_vars %}
- - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
-{% endfor %}
- image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
- imagePullPolicy: {{ k8s_image_pull_policy }}
- resources:
- limits:
- cpu: {{ cilium_cpu_limit }}
- memory: {{ cilium_memory_limit }}
- requests:
- cpu: {{ cilium_cpu_requests }}
- memory: {{ cilium_memory_requests }}
- lifecycle:
- postStart:
- exec:
- command:
- - "/cni-install.sh"
-{% if cilium_version | regex_replace('v') is version('1.10', '>=') %}
- - "--cni-exclusive={{ cilium_cni_exclusive }}"
-{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
- - "--log-file={{ cilium_cni_log_file }}"
-{% endif %}
- preStop:
- exec:
- command:
- - /cni-uninstall.sh
startupProbe:
httpGet:
host: '127.0.0.1'
@@ -131,80 +83,109 @@ spec:
successThreshold: 1
failureThreshold: 3
timeoutSeconds: 5
- name: cilium-agent
+ env:
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_K8S_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ - name: CILIUM_CLUSTERMESH_CONFIG
+ value: /var/lib/cilium/clustermesh/
+{% if cilium_kube_proxy_replacement == 'strict' %}
+ - name: KUBERNETES_SERVICE_HOST
+ value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+ - name: KUBERNETES_SERVICE_PORT
+ value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
+{% for env_var in cilium_agent_extra_env_vars %}
+ - {{ env_var | to_nice_yaml(indent=2) | indent(10) }}
+{% endfor %}
+ lifecycle:
+ postStart:
+ exec:
+ command:
+ - "/cni-install.sh"
+ - "--cni-exclusive={{ cilium_cni_exclusive | string | lower }}"
+{% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
+ - "--enable-debug={{ cilium_debug | string | lower }}"
+ - "--log-file={{ cilium_cni_log_file }}"
+{% endif %}
+ preStop:
+ exec:
+ command:
+ - /cni-uninstall.sh
+ resources:
+ limits:
+ cpu: {{ cilium_cpu_limit }}
+ memory: {{ cilium_memory_limit }}
+ requests:
+ cpu: {{ cilium_cpu_requests }}
+ memory: {{ cilium_memory_requests }}
{% if cilium_enable_prometheus or cilium_enable_hubble_metrics %}
ports:
{% endif %}
{% if cilium_enable_prometheus %}
- - containerPort: 9090
- hostPort: 9090
- name: prometheus
+ - name: prometheus
+ containerPort: {{ cilium_agent_scrape_port }}
+ hostPort: {{ cilium_agent_scrape_port }}
protocol: TCP
{% endif %}
{% if cilium_enable_hubble_metrics %}
- - containerPort: 9091
- hostPort: 9091
- name: hubble-metrics
+ - name: hubble-metrics
+ containerPort: {{ cilium_hubble_scrape_port }}
+ hostPort: {{ cilium_hubble_scrape_port }}
protocol: TCP
{% endif %}
securityContext:
privileged: true
volumeMounts:
- - mountPath: /sys/fs/bpf
- name: bpf-maps
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
mountPropagation: Bidirectional
- - mountPath: /var/run/cilium
- name: cilium-run
- - mountPath: /host/opt/cni/bin
- name: cni-path
- - mountPath: /host/etc/cni/net.d
- name: etc-cni-netd
-# pkg/workloads was depreca, removed in 1.7
-# https://github.com/cilium/cilium/pull/9447
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-{% if container_manager == 'docker' %}
- - mountPath: /var/run/docker.sock
- name: docker-socket
- readOnly: true
-{% else %}
- - name: "{{ container_manager }}-socket"
- mountPath: {{ cri_socket }}
- readOnly: true
-{% endif %}
-{% endif %}
+ - name: cilium-run
+ mountPath: /var/run/cilium
+ - name: cni-path
+ mountPath: /host/opt/cni/bin
+ - name: etc-cni-netd
+ mountPath: /host/etc/cni/net.d
{% if cilium_identity_allocation_mode == "kvstore" %}
- - mountPath: /var/lib/etcd-config
- name: etcd-config-path
+ - name: etcd-config-path
+ mountPath: /var/lib/etcd-config
readOnly: true
- - mountPath: "{{cilium_cert_dir}}"
- name: etcd-secrets
+ - name: etcd-secrets
+ mountPath: "{{cilium_cert_dir}}"
readOnly: true
{% endif %}
- - mountPath: /var/lib/cilium/clustermesh
- name: clustermesh-secrets
+ - name: clustermesh-secrets
+ mountPath: /var/lib/cilium/clustermesh
readOnly: true
- - mountPath: /tmp/cilium/config-map
- name: cilium-config-path
+ - name: cilium-config-path
+ mountPath: /tmp/cilium/config-map
readOnly: true
-{% if cilium_ip_masq_agent_enable %}
+{% if cilium_ip_masq_agent_enable %}\
- name: ip-masq-agent
mountPath: /etc/config
readOnly: true
{% endif %}
# Needed to be able to load kernel modules
- - mountPath: /lib/modules
- name: lib-modules
+ - name: lib-modules
+ mountPath: /lib/modules
readOnly: true
- - mountPath: /run/xtables.lock
- name: xtables-lock
+ - name: xtables-lock
+ mountPath: /run/xtables.lock
{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
- - mountPath: /etc/ipsec
- name: cilium-ipsec-secrets
+ - name: cilium-ipsec-secrets
+ mountPath: /etc/ipsec
readOnly: true
{% endif %}
{% if cilium_hubble_install %}
- - mountPath: /var/lib/cilium/tls/hubble
- name: hubble-tls
+ - name: hubble-tls
+ mountPath: /var/lib/cilium/tls/hubble
readOnly: true
{% endif %}
{% for volume_mount in cilium_agent_extra_volume_mounts %}
@@ -245,22 +226,50 @@ spec:
securityContext:
privileged: true
{% endif %}
- - command:
+{% if cilium_version | regex_replace('v') is version('1.11.7', '>=') %}
+ - name: apply-sysctl-overwrites
+ image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ env:
+ - name: BIN_PATH
+ value: /opt/cni/bin
+ command:
+ - sh
+ - -ec
+ # The statically linked Go program binary is invoked to avoid any
+ # dependency on utilities like sh that can be missing on certain
+ # distros installed on the underlying host. Copy the binary to the
+ # same directory where we install cilium cni plugin so that exec permissions
+ # are available.
+ - |
+ cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
+ nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
+ rm /hostbin/cilium-sysctlfix
+ volumeMounts:
+ - name: hostproc
+ mountPath: /hostproc
+ - name: cni-path
+ mountPath: /hostbin
+ securityContext:
+ privileged: true
+{% endif %}
+ - name: clean-cilium-state
+ image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ command:
- /init-container.sh
env:
- name: CILIUM_ALL_STATE
valueFrom:
configMapKeyRef:
- key: clean-cilium-state
name: cilium-config
+ key: clean-cilium-state
optional: true
- # CLEAN_CILIUM_BPF_STATE is deprecated in 1.6.
- # https://github.com/cilium/cilium/pull/7478
- - name: "{{ cilium_version | regex_replace('v') is version('1.6', '<')| ternary('CLEAN_CILIUM_BPF_STATE','CILIUM_BPF_STATE') }}"
+ - name: CILIUM_BPF_STATE
valueFrom:
configMapKeyRef:
- key: clean-cilium-bpf-state
name: cilium-config
+ key: clean-cilium-bpf-state
optional: true
# Removed in 1.11 and up.
# https://github.com/cilium/cilium/commit/f7a3f59fd74983c600bfce9cac364b76d20849d9
@@ -278,32 +287,25 @@ spec:
- name: KUBERNETES_SERVICE_PORT
value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
{% endif %}
-{% if cilium_version | regex_replace('v') is version('1.9', '<') %}
- image: "{{cilium_init_image_repo}}:{{cilium_init_image_tag}}"
-{% else %}
- image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
-{% endif %}
- imagePullPolicy: {{ k8s_image_pull_policy }}
- name: clean-cilium-state
securityContext:
privileged: true
volumeMounts:
- - mountPath: /sys/fs/bpf
- name: bpf-maps
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
# Required to mount cgroup filesystem from the host to cilium agent pod
- name: cilium-cgroup
mountPath: {{ cilium_cgroup_host_root }}
mountPropagation: HostToContainer
{% endif %}
- - mountPath: /var/run/cilium
- name: cilium-run
+ - name: cilium-run
+ mountPath: /var/run/cilium
resources:
requests:
cpu: 100m
memory: 100Mi
- priorityClassName: system-node-critical
restartPolicy: Always
+ priorityClassName: system-node-critical
serviceAccount: cilium
serviceAccountName: cilium
terminationGracePeriodSeconds: 1
@@ -312,36 +314,26 @@ spec:
{% if cilium_identity_allocation_mode == "kvstore" %}
dnsPolicy: ClusterFirstWithHostNet
{% endif %}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - topologyKey: kubernetes.io/hostname
+ labelSelector:
+ matchLabels:
+ k8s-app: cilium
tolerations:
- operator: Exists
volumes:
# To keep state between restarts / upgrades
- - hostPath:
+ - name: cilium-run
+ hostPath:
path: /var/run/cilium
type: DirectoryOrCreate
- name: cilium-run
# To keep state between restarts / upgrades for bpf maps
- - hostPath:
+ - name: bpf-maps
+ hostPath:
path: /sys/fs/bpf
type: DirectoryOrCreate
- name: bpf-maps
-# pkg/workloads was deprecated in 1.6, removed in 1.7
-# https://github.com/cilium/cilium/pull/9447
-{% if cilium_version | regex_replace('v') is version('1.7', '<') %}
-{% if container_manager == 'docker' %}
- # To read docker events from the node
- - hostPath:
- path: /var/run/docker.sock
- type: Socket
- name: docker-socket
-{% else %}
- # To read crio events from the node
- - hostPath:
- path: {{ cri_socket }}
- type: Socket
- name: {{ container_manager }}-socket
-{% endif %}
-{% endif %}
{% if cilium_version | regex_replace('v') is version('1.11', '>=') %}
# To mount cgroup2 filesystem on the host
- name: hostproc
@@ -355,34 +347,34 @@ spec:
type: DirectoryOrCreate
{% endif %}
# To install cilium cni plugin in the host
- - hostPath:
+ - name: cni-path
+ hostPath:
path: /opt/cni/bin
type: DirectoryOrCreate
- name: cni-path
# To install cilium cni configuration in the host
- - hostPath:
+ - name: etc-cni-netd
+ hostPath:
path: /etc/cni/net.d
type: DirectoryOrCreate
- name: etc-cni-netd
# To be able to load kernel modules
- - hostPath:
+ - name: lib-modules
+ hostPath:
path: /lib/modules
- name: lib-modules
# To access iptables concurrently with other processes (e.g. kube-proxy)
- - hostPath:
+ - name: xtables-lock
+ hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: xtables-lock
{% if cilium_identity_allocation_mode == "kvstore" %}
# To read the etcd config stored in config maps
- - configMap:
+ - name: etcd-config-path
+ configMap:
+ name: cilium-config
# note: the leading zero means this number is in octal representation: do not remove it
defaultMode: 0400
items:
- key: etcd-config
path: etcd.config
- name: cilium-config
- name: etcd-config-path
# To read the k8s etcd secrets in case the user might want to use TLS
- name: etcd-secrets
hostPath:
@@ -391,21 +383,22 @@ spec:
# To read the clustermesh configuration
- name: clustermesh-secrets
secret:
+ secretName: cilium-clustermesh
# note: the leading zero means this number is in octal representation: do not remove it
defaultMode: 0400
optional: true
- secretName: cilium-clustermesh
# To read the configuration from the config map
- - configMap:
+ - name: cilium-config-path
+ configMap:
name: cilium-config
- name: cilium-config-path
{% if cilium_ip_masq_agent_enable %}
- - configMap:
+ - name: ip-masq-agent
+ configMap:
name: ip-masq-agent
+ optional: true
items:
- key: config
path: ip-masq-agent
- name: ip-masq-agent
{% endif %}
{% if cilium_encryption_enabled and cilium_encryption_type == "ipsec" %}
- name: cilium-ipsec-secrets
@@ -420,21 +413,12 @@ spec:
sources:
- secret:
name: hubble-server-certs
+ optional: true
items:
+ - key: ca.crt
+ path: client-ca.crt
- key: tls.crt
path: server.crt
- key: tls.key
path: server.key
- optional: true
- - configMap:
- name: hubble-ca-cert
- items:
- - key: ca.crt
- path: client-ca.crt
- optional: true
{% endif %}
- updateStrategy:
- rollingUpdate:
- # Specifies the maximum number of Pods that can be unavailable during the update process.
- maxUnavailable: 2
- type: RollingUpdate
--
GitLab