From 6769bb32b1ebb109e8b3977d3dcb3df8c8cbafcf Mon Sep 17 00:00:00 2001
From: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>
Date: Fri, 3 Mar 2023 09:23:08 +0100
Subject: [PATCH] Network plugin custom (#9819)
* network_plugin/custom_cni: add CNI to apply provided manifests
Add a new simple custom_cni to install provided Kubernetes manifests.
This could be useful to use manifests directly provided by a CNI when
there are not support by Kubespray (i.e.: helm chart or any other manifests
generation method).
Co-authored-by: James Landrein <james.landrein@proton.ch>
Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>
* network_plugin/custom_cni: add test with cilium
Co-authored-by: James Landrein <james.landrein@proton.ch>
Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>
---------
Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>
Co-authored-by: James Landrein <james.landrein@proton.ch>
---
.ansible-lint | 3 +
.gitlab-ci/packet.yml | 5 +
.yamllint | 2 +
docs/ci.md | 102 +-
.../preinstall/tasks/0020-verify-settings.yml | 2 +-
.../custom_cni/defaults/main.yml | 3 +
.../network_plugin/custom_cni/tasks/main.yml | 27 +
roles/network_plugin/meta/main.yml | 5 +
tests/files/custom_cni/README.md | 11 +
tests/files/custom_cni/cilium.yaml | 1056 +++++++++++++++++
tests/files/custom_cni/values.yaml | 11 +
tests/files/packet_debian11-custom-cni.yml | 9 +
12 files changed, 1184 insertions(+), 52 deletions(-)
create mode 100644 roles/network_plugin/custom_cni/defaults/main.yml
create mode 100644 roles/network_plugin/custom_cni/tasks/main.yml
create mode 100644 tests/files/custom_cni/README.md
create mode 100644 tests/files/custom_cni/cilium.yaml
create mode 100644 tests/files/custom_cni/values.yaml
create mode 100644 tests/files/packet_debian11-custom-cni.yml
diff --git a/.ansible-lint b/.ansible-lint
index 5848a6034..d84419e6a 100644
--- a/.ansible-lint
+++ b/.ansible-lint
@@ -35,3 +35,6 @@ skip_list:
# Roles in kubespray don't need fully qualified collection names
# (Disabled in Feb 2023)
- 'fqcn-builtins'
+exclude_paths:
+ # Generated files
+ - tests/files/custom_cni/cilium.yaml
diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml
index f141f6b6f..a30b96d6a 100644
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -268,6 +268,11 @@ packet_fedora36-kube-ovn:
extends: .packet_periodic
when: on_success
+packet_debian11-custom-cni:
+ stage: deploy-part2
+ extends: .packet_pr
+ when: manual
+
# ### PR JOBS PART3
# Long jobs (45min+)
diff --git a/.yamllint b/.yamllint
index 01d8b333b..8a6245d1b 100644
--- a/.yamllint
+++ b/.yamllint
@@ -3,6 +3,8 @@ extends: default
ignore: |
.git/
+ # Generated file
+ tests/files/custom_cni/cilium.yaml
rules:
braces:
diff --git a/docs/ci.md b/docs/ci.md
index f953aab9d..405552590 100644
--- a/docs/ci.md
+++ b/docs/ci.md
@@ -4,60 +4,60 @@ To generate this Matrix run `./tests/scripts/md-table/main.py`
## containerd
-| OS / CNI | calico | canal | cilium | flannel | kube-ovn | kube-router | macvlan | weave |
-|---| --- | --- | --- | --- | --- | --- | --- | --- |
-almalinux8 | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
-amazon | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 | :white_check_mark: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
-debian10 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-debian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
-fedora35 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
-fedora36 | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
-opensuse | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux9 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: |
-ubuntu18 | :white_check_mark: | :x: | :white_check_mark: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
-ubuntu20 | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
-ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+| OS / CNI | calico | canal | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+amazon | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 | :white_check_mark: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: |
+debian10 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
+debian9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: |
+fedora35 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: |
+fedora36 | :x: | :x: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: |
+opensuse | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux9 | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 | :x: | :white_check_mark: | :x: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: |
+ubuntu18 | :white_check_mark: | :x: | :white_check_mark: | :x: | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: |
+ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: |
+ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
## crio
-| OS / CNI | calico | canal | cilium | flannel | kube-ovn | kube-router | macvlan | weave |
-|---| --- | --- | --- | --- | --- | --- | --- | --- |
-almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian11 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora35 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora36 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-opensuse | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu18 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+| OS / CNI | calico | canal | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora35 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora36 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+opensuse | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu18 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
## docker
-| OS / CNI | calico | canal | cilium | flannel | kube-ovn | kube-router | macvlan | weave |
-|---| --- | --- | --- | --- | --- | --- | --- | --- |
-almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-centos7 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian10 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-debian9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora35 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-fedora36 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
-opensuse | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: |
-rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu16 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
-ubuntu18 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
-ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+| OS / CNI | calico | canal | cilium | custom_cni | flannel | kube-ovn | kube-router | macvlan | weave |
+|---| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+almalinux8 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+amazon | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+centos7 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian10 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian11 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+debian9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora35 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+fedora36 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+opensuse | :x: | :x: | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux8 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+rockylinux9 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu16 | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :white_check_mark: |
+ubuntu18 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu20 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
+ubuntu22 | :white_check_mark: | :x: | :x: | :x: | :x: | :x: | :x: | :x: | :x: |
diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
index 6ceafbec9..99f4316ab 100644
--- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
+++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
@@ -30,7 +30,7 @@
- name: Stop if unknown network plugin
assert:
- that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni','kube-ovn', 'kube-router', 'macvlan']
+ that: kube_network_plugin in ['calico', 'canal', 'flannel', 'weave', 'cloud', 'cilium', 'cni', 'kube-ovn', 'kube-router', 'macvlan', 'custom_cni']
msg: "{{ kube_network_plugin }} is not supported"
when:
- kube_network_plugin is defined
diff --git a/roles/network_plugin/custom_cni/defaults/main.yml b/roles/network_plugin/custom_cni/defaults/main.yml
new file mode 100644
index 000000000..5cde372d5
--- /dev/null
+++ b/roles/network_plugin/custom_cni/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+
+custom_cni_manifests: []
diff --git a/roles/network_plugin/custom_cni/tasks/main.yml b/roles/network_plugin/custom_cni/tasks/main.yml
new file mode 100644
index 000000000..2dfba28cb
--- /dev/null
+++ b/roles/network_plugin/custom_cni/tasks/main.yml
@@ -0,0 +1,27 @@
+---
+
+- name: Cilium | Check Cilium encryption `cilium_ipsec_key` for ipsec
+ assert:
+ that:
+ - "custom_cni_manifests | length > 0"
+ msg: "custom_cni_manifests should not be empty"
+
+- name: Custom CNI | Copy Custom manifests
+ template:
+ src: "{{ item }}"
+ dest: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}"
+ mode: 0644
+ loop: "{{ custom_cni_manifests }}"
+ delegate_to: "{{ groups['kube_control_plane'] | first }}"
+ run_once: true
+
+- name: Custom CNI | Start Resources
+ kube:
+ namespace: "kube-system"
+ kubectl: "{{ bin_dir }}/kubectl"
+ filename: "{{ kube_config_dir }}/{{ item | basename | replace('.j2', '') }}"
+ state: "latest"
+ wait: true
+ loop: "{{ custom_cni_manifests }}"
+ delegate_to: "{{ groups['kube_control_plane'] | first }}"
+ run_once: true
diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml
index cb013fcca..fd20c5cbc 100644
--- a/roles/network_plugin/meta/main.yml
+++ b/roles/network_plugin/meta/main.yml
@@ -42,6 +42,11 @@ dependencies:
tags:
- kube-router
+ - role: network_plugin/custom_cni
+ when: kube_network_plugin == 'custom_cni'
+ tags:
+ - custom_cni
+
- role: network_plugin/multus
when: kube_network_plugin_multus
tags:
diff --git a/tests/files/custom_cni/README.md b/tests/files/custom_cni/README.md
new file mode 100644
index 000000000..dd3024061
--- /dev/null
+++ b/tests/files/custom_cni/README.md
@@ -0,0 +1,11 @@
+# Custom CNI manifest generation
+
+As an example we are using Cilium for testing the network_plugins/custom_cni.
+
+To update the generated manifests to the latest version do the following:
+
+```sh
+helm repo add cilium https://helm.cilium.io/
+helm repo update
+helm template cilium/cilium -n kube-system -f values.yaml > cilium.yaml
+```
diff --git a/tests/files/custom_cni/cilium.yaml b/tests/files/custom_cni/cilium.yaml
new file mode 100644
index 000000000..9bd3bfbe4
--- /dev/null
+++ b/tests/files/custom_cni/cilium.yaml
@@ -0,0 +1,1056 @@
+---
+# Source: cilium/templates/cilium-agent/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: "cilium"
+ namespace: kube-system
+---
+# Source: cilium/templates/cilium-operator/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: "cilium-operator"
+ namespace: kube-system
+---
+# Source: cilium/templates/cilium-configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: cilium-config
+ namespace: kube-system
+data:
+
+ # Identity allocation mode selects how identities are shared between cilium
+ # nodes by setting how they are stored. The options are "crd" or "kvstore".
+ # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+ # These can be queried with:
+ # kubectl get ciliumid
+ # - "kvstore" stores identities in an etcd kvstore, that is
+ # configured below. Cilium versions before 1.6 supported only the kvstore
+ # backend. Upgrades from these older cilium versions should continue using
+ # the kvstore by commenting out the identity-allocation-mode below, or
+ # setting it to "kvstore".
+ identity-allocation-mode: crd
+ identity-heartbeat-timeout: "30m0s"
+ identity-gc-interval: "15m0s"
+ cilium-endpoint-gc-interval: "5m0s"
+ nodes-gc-interval: "5m0s"
+ skip-cnp-status-startup-clean: "false"
+ # Disable the usage of CiliumEndpoint CRD
+ disable-endpoint-crd: "false"
+
+ # If you want to run cilium in debug mode change this value to true
+ debug: "false"
+ debug-verbose: ""
+ # The agent can be put into the following three policy enforcement modes
+ # default, always and never.
+ # https://docs.cilium.io/en/latest/security/policy/intro/#policy-enforcement-modes
+ enable-policy: "default"
+
+ # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
+ # address.
+ enable-ipv4: "true"
+
+ # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
+ # address.
+ enable-ipv6: "false"
+ # Users who wish to specify their own custom CNI configuration file must set
+ # custom-cni-conf to "true", otherwise Cilium may overwrite the configuration.
+ custom-cni-conf: "false"
+ enable-bpf-clock-probe: "true"
+ # If you want cilium monitor to aggregate tracing for packets, set this level
+ # to "low", "medium", or "maximum". The higher the level, the less packets
+ # that will be seen in monitor output.
+ monitor-aggregation: medium
+
+ # The monitor aggregation interval governs the typical time between monitor
+ # notification events for each allowed connection.
+ #
+ # Only effective when monitor aggregation is set to "medium" or higher.
+ monitor-aggregation-interval: "5s"
+
+ # The monitor aggregation flags determine which TCP flags which, upon the
+ # first observation, cause monitor notifications to be generated.
+ #
+ # Only effective when monitor aggregation is set to "medium" or higher.
+ monitor-aggregation-flags: all
+ # Specifies the ratio (0.0-1.0] of total system memory to use for dynamic
+ # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+ bpf-map-dynamic-size-ratio: "0.0025"
+ # bpf-policy-map-max specifies the maximum number of entries in endpoint
+ # policy map (per endpoint)
+ bpf-policy-map-max: "16384"
+ # bpf-lb-map-max specifies the maximum number of entries in bpf lb service,
+ # backend and affinity maps.
+ bpf-lb-map-max: "65536"
+ bpf-lb-external-clusterip: "false"
+
+ # Pre-allocation of map entries allows per-packet latency to be reduced, at
+ # the expense of up-front memory allocation for the entries in the maps. The
+ # default value below will minimize memory usage in the default installation;
+ # users who are sensitive to latency may consider setting this to "true".
+ #
+ # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+ # this option and behave as though it is set to "true".
+ #
+ # If this value is modified, then during the next Cilium startup the restore
+ # of existing endpoints and tracking of ongoing connections may be disrupted.
+ # As a result, reply packets may be dropped and the load-balancing decisions
+ # for established connections may change.
+ #
+ # If this option is set to "false" during an upgrade from 1.3 or earlier to
+ # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+ preallocate-bpf-maps: "false"
+
+ # Regular expression matching compatible Istio sidecar istio-proxy
+ # container image names
+ sidecar-istio-proxy-image: "cilium/istio_proxy"
+
+ # Name of the cluster. Only relevant when building a mesh of clusters.
+ cluster-name: default
+ # Unique ID of the cluster. Must be unique across all conneted clusters and
+ # in the range of 1 and 255. Only relevant when building a mesh of clusters.
+ cluster-id: "0"
+
+ # Encapsulation mode for communication between nodes
+ # Possible values:
+ # - disabled
+ # - vxlan (default)
+ # - geneve
+ tunnel: "vxlan"
+
+
+ # Enables L7 proxy for L7 policy enforcement and visibility
+ enable-l7-proxy: "true"
+
+ enable-ipv4-masquerade: "true"
+ enable-ipv6-big-tcp: "false"
+ enable-ipv6-masquerade: "true"
+
+ enable-xt-socket-fallback: "true"
+ install-iptables-rules: "true"
+ install-no-conntrack-iptables-rules: "false"
+
+ auto-direct-node-routes: "false"
+ enable-local-redirect-policy: "false"
+
+ kube-proxy-replacement: "disabled"
+ bpf-lb-sock: "false"
+ enable-health-check-nodeport: "true"
+ node-port-bind-protection: "true"
+ enable-auto-protect-node-port-range: "true"
+ enable-svc-source-range-check: "true"
+ enable-l2-neigh-discovery: "true"
+ arping-refresh-period: "30s"
+ enable-endpoint-health-checking: "true"
+ enable-health-checking: "true"
+ enable-well-known-identities: "false"
+ enable-remote-node-identity: "true"
+ synchronize-k8s-nodes: "true"
+ operator-api-serve-addr: "127.0.0.1:9234"
+ ipam: "cluster-pool"
+ cluster-pool-ipv4-cidr: "{{ kube_pods_subnet }}"
+ cluster-pool-ipv4-mask-size: "24"
+ disable-cnp-status-updates: "true"
+ enable-vtep: "false"
+ vtep-endpoint: ""
+ vtep-cidr: ""
+ vtep-mask: ""
+ vtep-mac: ""
+ enable-bgp-control-plane: "false"
+ procfs: "/host/proc"
+ bpf-root: "/sys/fs/bpf"
+ cgroup-root: "/run/cilium/cgroupv2"
+ enable-k8s-terminating-endpoint: "true"
+ enable-sctp: "false"
+ remove-cilium-node-taints: "true"
+ set-cilium-is-up-condition: "true"
+ unmanaged-pod-watcher-interval: "15"
+ tofqdns-dns-reject-response-code: "refused"
+ tofqdns-enable-dns-compression: "true"
+ tofqdns-endpoint-max-ip-per-hostname: "50"
+ tofqdns-idle-connection-grace-period: "0s"
+ tofqdns-max-deferred-connection-deletes: "10000"
+ tofqdns-min-ttl: "3600"
+ tofqdns-proxy-response-max-delay: "100ms"
+ agent-not-ready-taint-key: "node.cilium.io/agent-not-ready"
+---
+# Source: cilium/templates/cilium-agent/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: cilium
+ labels:
+ app.kubernetes.io/part-of: cilium
+rules:
+- apiGroups:
+ - networking.k8s.io
+ resources:
+ - networkpolicies
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - discovery.k8s.io
+ resources:
+ - endpointslices
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ - namespaces
+ - services
+ - pods
+ - endpoints
+ - nodes
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - apiextensions.k8s.io
+ resources:
+ - customresourcedefinitions
+ verbs:
+ - list
+ - watch
+ # This is used when validating policies in preflight. This will need to stay
+ # until we figure out how to avoid "get" inside the preflight, and then
+ # should be removed ideally.
+ - get
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumloadbalancerippools
+ - ciliumbgppeeringpolicies
+ - ciliumclusterwideenvoyconfigs
+ - ciliumclusterwidenetworkpolicies
+ - ciliumegressgatewaypolicies
+ - ciliumendpoints
+ - ciliumendpointslices
+ - ciliumenvoyconfigs
+ - ciliumidentities
+ - ciliumlocalredirectpolicies
+ - ciliumnetworkpolicies
+ - ciliumnodes
+ - ciliumnodeconfigs
+ verbs:
+ - list
+ - watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumidentities
+ - ciliumendpoints
+ - ciliumnodes
+ verbs:
+ - create
+- apiGroups:
+ - cilium.io
+ # To synchronize garbage collection of such resources
+ resources:
+ - ciliumidentities
+ verbs:
+ - update
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumendpoints
+ verbs:
+ - delete
+ - get
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnodes
+ - ciliumnodes/status
+ verbs:
+ - get
+ - update
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnetworkpolicies/status
+ - ciliumclusterwidenetworkpolicies/status
+ - ciliumendpoints/status
+ - ciliumendpoints
+ verbs:
+ - patch
+---
+# Source: cilium/templates/cilium-operator/clusterrole.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+ name: cilium-operator
+ labels:
+ app.kubernetes.io/part-of: cilium
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - get
+ - list
+ - watch
+ # to automatically delete [core|kube]dns pods so that are starting to being
+ # managed by Cilium
+ - delete
+- apiGroups:
+ - ""
+ resources:
+ - nodes
+ verbs:
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ # To remove node taints
+ - nodes
+ # To set NetworkUnavailable false on startup
+ - nodes/status
+ verbs:
+ - patch
+- apiGroups:
+ - discovery.k8s.io
+ resources:
+ - endpointslices
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ # to perform LB IP allocation for BGP
+ - services/status
+ verbs:
+ - update
+ - patch
+- apiGroups:
+ - ""
+ resources:
+ # to check apiserver connectivity
+ - namespaces
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - ""
+ resources:
+ # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+ - services
+ - endpoints
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnetworkpolicies
+ - ciliumclusterwidenetworkpolicies
+ verbs:
+ # Create auto-generated CNPs and CCNPs from Policies that have 'toGroups'
+ - create
+ - update
+ - deletecollection
+ # To update the status of the CNPs and CCNPs
+ - patch
+ - get
+ - list
+ - watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnetworkpolicies/status
+ - ciliumclusterwidenetworkpolicies/status
+ verbs:
+ # Update the auto-generated CNPs and CCNPs status.
+ - patch
+ - update
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumendpoints
+ - ciliumidentities
+ verbs:
+ # To perform garbage collection of such resources
+ - delete
+ - list
+ - watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumidentities
+ verbs:
+ # To synchronize garbage collection of such resources
+ - update
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnodes
+ verbs:
+ - create
+ - update
+ - get
+ - list
+ - watch
+ # To perform CiliumNode garbage collector
+ - delete
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumnodes/status
+ verbs:
+ - update
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumendpointslices
+ - ciliumenvoyconfigs
+ verbs:
+ - create
+ - update
+ - get
+ - list
+ - watch
+ - delete
+ - patch
+- apiGroups:
+ - apiextensions.k8s.io
+ resources:
+ - customresourcedefinitions
+ verbs:
+ - create
+ - get
+ - list
+ - watch
+- apiGroups:
+ - apiextensions.k8s.io
+ resources:
+ - customresourcedefinitions
+ verbs:
+ - update
+ resourceNames:
+ - ciliumloadbalancerippools.cilium.io
+ - ciliumbgppeeringpolicies.cilium.io
+ - ciliumclusterwideenvoyconfigs.cilium.io
+ - ciliumclusterwidenetworkpolicies.cilium.io
+ - ciliumegressgatewaypolicies.cilium.io
+ - ciliumendpoints.cilium.io
+ - ciliumendpointslices.cilium.io
+ - ciliumenvoyconfigs.cilium.io
+ - ciliumexternalworkloads.cilium.io
+ - ciliumidentities.cilium.io
+ - ciliumlocalredirectpolicies.cilium.io
+ - ciliumnetworkpolicies.cilium.io
+ - ciliumnodes.cilium.io
+ - ciliumnodeconfigs.cilium.io
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumloadbalancerippools
+ verbs:
+ - get
+ - list
+ - watch
+- apiGroups:
+ - cilium.io
+ resources:
+ - ciliumloadbalancerippools/status
+ verbs:
+ - patch
+# For cilium-operator running in HA mode.
+#
+# Cilium operator running in HA mode requires the use of ResourceLock for Leader Election
+# between multiple running instances.
+# The preferred way of doing this is to use LeasesResourceLock as edits to Leases are less
+# common and fewer objects in the cluster watch "all Leases".
+- apiGroups:
+ - coordination.k8s.io
+ resources:
+ - leases
+ verbs:
+ - create
+ - get
+ - update
+---
+# Source: cilium/templates/cilium-agent/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: cilium
+ labels:
+ app.kubernetes.io/part-of: cilium
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cilium
+subjects:
+- kind: ServiceAccount
+ name: "cilium"
+ namespace: kube-system
+---
+# Source: cilium/templates/cilium-operator/clusterrolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: cilium-operator
+ labels:
+ app.kubernetes.io/part-of: cilium
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cilium-operator
+subjects:
+- kind: ServiceAccount
+ name: "cilium-operator"
+ namespace: kube-system
+---
+# Source: cilium/templates/cilium-agent/role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: cilium-config-agent
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/part-of: cilium
+rules:
+- apiGroups:
+ - ""
+ resources:
+ - configmaps
+ verbs:
+ - get
+ - list
+ - watch
+---
+# Source: cilium/templates/cilium-agent/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: cilium-config-agent
+ namespace: kube-system
+ labels:
+ app.kubernetes.io/part-of: cilium
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: cilium-config-agent
+subjects:
+ - kind: ServiceAccount
+ name: "cilium"
+ namespace: kube-system
+---
+# Source: cilium/templates/cilium-agent/daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+ name: cilium
+ namespace: kube-system
+ labels:
+ k8s-app: cilium
+ app.kubernetes.io/part-of: cilium
+ app.kubernetes.io/name: cilium-agent
+spec:
+ selector:
+ matchLabels:
+ k8s-app: cilium
+ updateStrategy:
+ rollingUpdate:
+ maxUnavailable: 2
+ type: RollingUpdate
+ template:
+ metadata:
+ annotations:
+ # Set app AppArmor's profile to "unconfined". The value of this annotation
+ # can be modified as long users know which profiles they have available
+ # in AppArmor.
+ container.apparmor.security.beta.kubernetes.io/cilium-agent: "unconfined"
+ container.apparmor.security.beta.kubernetes.io/clean-cilium-state: "unconfined"
+ container.apparmor.security.beta.kubernetes.io/mount-cgroup: "unconfined"
+ container.apparmor.security.beta.kubernetes.io/apply-sysctl-overwrites: "unconfined"
+ labels:
+ k8s-app: cilium
+ app.kubernetes.io/name: cilium-agent
+ app.kubernetes.io/part-of: cilium
+ spec:
+ containers:
+ - name: cilium-agent
+ image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ imagePullPolicy: IfNotPresent
+ command:
+ - cilium-agent
+ args:
+ - --config-dir=/tmp/cilium/config-map
+ startupProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9879
+ scheme: HTTP
+ httpHeaders:
+ - name: "brief"
+ value: "true"
+ failureThreshold: 105
+ periodSeconds: 2
+ successThreshold: 1
+ livenessProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9879
+ scheme: HTTP
+ httpHeaders:
+ - name: "brief"
+ value: "true"
+ periodSeconds: 30
+ successThreshold: 1
+ failureThreshold: 10
+ timeoutSeconds: 5
+ readinessProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9879
+ scheme: HTTP
+ httpHeaders:
+ - name: "brief"
+ value: "true"
+ periodSeconds: 30
+ successThreshold: 1
+ failureThreshold: 3
+ timeoutSeconds: 5
+ env:
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_K8S_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ - name: CILIUM_CLUSTERMESH_CONFIG
+ value: /var/lib/cilium/clustermesh/
+ - name: CILIUM_CNI_CHAINING_MODE
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: cni-chaining-mode
+ optional: true
+ - name: CILIUM_CUSTOM_CNI_CONF
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: custom-cni-conf
+ optional: true
+ lifecycle:
+ postStart:
+ exec:
+ command:
+ - "/cni-install.sh"
+ - "--enable-debug=false"
+ - "--cni-exclusive=true"
+ - "--log-file=/var/run/cilium/cilium-cni.log"
+ preStop:
+ exec:
+ command:
+ - /cni-uninstall.sh
+ securityContext:
+ seLinuxOptions:
+ level: s0
+ type: spc_t
+ capabilities:
+ add:
+ - CHOWN
+ - KILL
+ - NET_ADMIN
+ - NET_RAW
+ - IPC_LOCK
+ - SYS_MODULE
+ - SYS_ADMIN
+ - SYS_RESOURCE
+ - DAC_OVERRIDE
+ - FOWNER
+ - SETGID
+ - SETUID
+ drop:
+ - ALL
+ terminationMessagePolicy: FallbackToLogsOnError
+ volumeMounts:
+ # Unprivileged containers need to mount /proc/sys/net from the host
+ # to have write access
+ - mountPath: /host/proc/sys/net
+ name: host-proc-sys-net
+ # Unprivileged containers need to mount /proc/sys/kernel from the host
+ # to have write access
+ - mountPath: /host/proc/sys/kernel
+ name: host-proc-sys-kernel
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
+ # Unprivileged containers can't set mount propagation to bidirectional
+ # in this case we will mount the bpf fs from an init container that
+ # is privileged and set the mount propagation from host to container
+ # in Cilium.
+ mountPropagation: HostToContainer
+ - name: cilium-run
+ mountPath: /var/run/cilium
+ - name: cni-path
+ mountPath: /host/opt/cni/bin
+ - name: etc-cni-netd
+ mountPath: /host/etc/cni/net.d
+ - name: clustermesh-secrets
+ mountPath: /var/lib/cilium/clustermesh
+ readOnly: true
+ # Needed to be able to load kernel modules
+ - name: lib-modules
+ mountPath: /lib/modules
+ readOnly: true
+ - name: xtables-lock
+ mountPath: /run/xtables.lock
+ - name: tmp
+ mountPath: /tmp
+ initContainers:
+ - name: config
+ image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ imagePullPolicy: IfNotPresent
+ command:
+ - cilium
+ - build-config
+ env:
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_K8S_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - name: tmp
+ mountPath: /tmp
+ terminationMessagePolicy: FallbackToLogsOnError
+ # Required to mount cgroup2 filesystem on the underlying Kubernetes node.
+ # We use nsenter command with host's cgroup and mount namespaces enabled.
+ - name: mount-cgroup
+ image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: CGROUP_ROOT
+ value: /run/cilium/cgroupv2
+ - name: BIN_PATH
+ value: /opt/cni/bin
+ command:
+ - sh
+ - -ec
+ # The statically linked Go program binary is invoked to avoid any
+ # dependency on utilities like sh and mount that can be missing on certain
+ # distros installed on the underlying host. Copy the binary to the
+ # same directory where we install cilium cni plugin so that exec permissions
+ # are available.
+ - |
+ cp /usr/bin/cilium-mount /hostbin/cilium-mount;
+ nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-mount" $CGROUP_ROOT;
+ rm /hostbin/cilium-mount
+ volumeMounts:
+ - name: hostproc
+ mountPath: /hostproc
+ - name: cni-path
+ mountPath: /hostbin
+ terminationMessagePolicy: FallbackToLogsOnError
+ securityContext:
+ seLinuxOptions:
+ level: s0
+ type: spc_t
+ capabilities:
+ add:
+ - SYS_ADMIN
+ - SYS_CHROOT
+ - SYS_PTRACE
+ drop:
+ - ALL
+ - name: apply-sysctl-overwrites
+ image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: BIN_PATH
+ value: /opt/cni/bin
+ command:
+ - sh
+ - -ec
+ # The statically linked Go program binary is invoked to avoid any
+ # dependency on utilities like sh that can be missing on certain
+ # distros installed on the underlying host. Copy the binary to the
+ # same directory where we install cilium cni plugin so that exec permissions
+ # are available.
+ - |
+ cp /usr/bin/cilium-sysctlfix /hostbin/cilium-sysctlfix;
+ nsenter --mount=/hostproc/1/ns/mnt "${BIN_PATH}/cilium-sysctlfix";
+ rm /hostbin/cilium-sysctlfix
+ volumeMounts:
+ - name: hostproc
+ mountPath: /hostproc
+ - name: cni-path
+ mountPath: /hostbin
+ terminationMessagePolicy: FallbackToLogsOnError
+ securityContext:
+ seLinuxOptions:
+ level: s0
+ type: spc_t
+ capabilities:
+ add:
+ - SYS_ADMIN
+ - SYS_CHROOT
+ - SYS_PTRACE
+ drop:
+ - ALL
+ # Mount the bpf fs if it is not mounted. We will perform this task
+ # from a privileged container because the mount propagation bidirectional
+ # only works from privileged containers.
+ - name: mount-bpf-fs
+ image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ imagePullPolicy: IfNotPresent
+ args:
+ - 'mount | grep "/sys/fs/bpf type bpf" || mount -t bpf bpf /sys/fs/bpf'
+ command:
+ - /bin/bash
+ - -c
+ - --
+ terminationMessagePolicy: FallbackToLogsOnError
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
+ mountPropagation: Bidirectional
+ - name: clean-cilium-state
+ image: "quay.io/cilium/cilium:v1.13.0@sha256:6544a3441b086a2e09005d3e21d1a4afb216fae19c5a60b35793c8a9438f8f68"
+ imagePullPolicy: IfNotPresent
+ command:
+ - /init-container.sh
+ env:
+ - name: CILIUM_ALL_STATE
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: clean-cilium-state
+ optional: true
+ - name: CILIUM_BPF_STATE
+ valueFrom:
+ configMapKeyRef:
+ name: cilium-config
+ key: clean-cilium-bpf-state
+ optional: true
+ terminationMessagePolicy: FallbackToLogsOnError
+ securityContext:
+ seLinuxOptions:
+ level: s0
+ type: spc_t
+ capabilities:
+ add:
+ - NET_ADMIN
+ - SYS_MODULE
+ - SYS_ADMIN
+ - SYS_RESOURCE
+ drop:
+ - ALL
+ volumeMounts:
+ - name: bpf-maps
+ mountPath: /sys/fs/bpf
+ # Required to mount cgroup filesystem from the host to cilium agent pod
+ - name: cilium-cgroup
+ mountPath: /run/cilium/cgroupv2
+ mountPropagation: HostToContainer
+ - name: cilium-run
+ mountPath: /var/run/cilium
+ resources:
+ requests:
+ cpu: 100m
+ memory: 100Mi # wait-for-kube-proxy
+ restartPolicy: Always
+ priorityClassName: system-node-critical
+ serviceAccount: "cilium"
+ serviceAccountName: "cilium"
+ terminationGracePeriodSeconds: 1
+ hostNetwork: true
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ k8s-app: cilium
+ topologyKey: kubernetes.io/hostname
+ nodeSelector:
+ kubernetes.io/os: linux
+ tolerations:
+ - operator: Exists
+ volumes:
+ # For sharing configuration between the "config" initContainer and the agent
+ - name: tmp
+ emptyDir: {}
+ # To keep state between restarts / upgrades
+ - name: cilium-run
+ hostPath:
+ path: /var/run/cilium
+ type: DirectoryOrCreate
+ # To keep state between restarts / upgrades for bpf maps
+ - name: bpf-maps
+ hostPath:
+ path: /sys/fs/bpf
+ type: DirectoryOrCreate
+ # To mount cgroup2 filesystem on the host
+ - name: hostproc
+ hostPath:
+ path: /proc
+ type: Directory
+ # To keep state between restarts / upgrades for cgroup2 filesystem
+ - name: cilium-cgroup
+ hostPath:
+ path: /run/cilium/cgroupv2
+ type: DirectoryOrCreate
+ # To install cilium cni plugin in the host
+ - name: cni-path
+ hostPath:
+ path: /opt/cni/bin
+ type: DirectoryOrCreate
+ # To install cilium cni configuration in the host
+ - name: etc-cni-netd
+ hostPath:
+ path: /etc/cni/net.d
+ type: DirectoryOrCreate
+ # To be able to load kernel modules
+ - name: lib-modules
+ hostPath:
+ path: /lib/modules
+ # To access iptables concurrently with other processes (e.g. kube-proxy)
+ - name: xtables-lock
+ hostPath:
+ path: /run/xtables.lock
+ type: FileOrCreate
+ # To read the clustermesh configuration
+ - name: clustermesh-secrets
+ secret:
+ secretName: cilium-clustermesh
+ # note: the leading zero means this number is in octal representation: do not remove it
+ defaultMode: 0400
+ optional: true
+ - name: host-proc-sys-net
+ hostPath:
+ path: /proc/sys/net
+ type: Directory
+ - name: host-proc-sys-kernel
+ hostPath:
+ path: /proc/sys/kernel
+ type: Directory
+---
+# Source: cilium/templates/cilium-operator/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: cilium-operator
+ namespace: kube-system
+ labels:
+ io.cilium/app: operator
+ name: cilium-operator
+ app.kubernetes.io/part-of: cilium
+ app.kubernetes.io/name: cilium-operator
+spec:
+ # See docs on ServerCapabilities.LeasesResourceLock in file pkg/k8s/version/version.go
+ # for more details.
+ replicas: 2
+ selector:
+ matchLabels:
+ io.cilium/app: operator
+ name: cilium-operator
+ strategy:
+ rollingUpdate:
+ maxSurge: 1
+ maxUnavailable: 1
+ type: RollingUpdate
+ template:
+ metadata:
+ annotations:
+ labels:
+ io.cilium/app: operator
+ name: cilium-operator
+ app.kubernetes.io/part-of: cilium
+ app.kubernetes.io/name: cilium-operator
+ spec:
+ containers:
+ - name: cilium-operator
+ image: "quay.io/cilium/operator-generic:v1.13.0@sha256:4b58d5b33e53378355f6e8ceb525ccf938b7b6f5384b35373f1f46787467ebf5"
+ imagePullPolicy: IfNotPresent
+ command:
+ - cilium-operator-generic
+ args:
+ - --config-dir=/tmp/cilium/config-map
+ - --debug=$(CILIUM_DEBUG)
+ env:
+ - name: K8S_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ - name: CILIUM_K8S_NAMESPACE
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: metadata.namespace
+ - name: CILIUM_DEBUG
+ valueFrom:
+ configMapKeyRef:
+ key: debug
+ name: cilium-config
+ optional: true
+ livenessProbe:
+ httpGet:
+ host: "127.0.0.1"
+ path: /healthz
+ port: 9234
+ scheme: HTTP
+ initialDelaySeconds: 60
+ periodSeconds: 10
+ timeoutSeconds: 3
+ volumeMounts:
+ - name: cilium-config-path
+ mountPath: /tmp/cilium/config-map
+ readOnly: true
+ terminationMessagePolicy: FallbackToLogsOnError
+ hostNetwork: true
+ restartPolicy: Always
+ priorityClassName: system-cluster-critical
+ serviceAccount: "cilium-operator"
+ serviceAccountName: "cilium-operator"
+ # In HA mode, cilium-operator pods must not be scheduled on the same
+ # node as they will clash with each other.
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchLabels:
+ io.cilium/app: operator
+ topologyKey: kubernetes.io/hostname
+ nodeSelector:
+ kubernetes.io/os: linux
+ tolerations:
+ - operator: Exists
+ volumes:
+ # To read the configuration from the config map
+ - name: cilium-config-path
+ configMap:
+ name: cilium-config
+---
+# Source: cilium/templates/cilium-secrets-namespace.yaml
+# Only create the namespace if it's different from Ingress secret namespace or Ingress is not enabled.
diff --git a/tests/files/custom_cni/values.yaml b/tests/files/custom_cni/values.yaml
new file mode 100644
index 000000000..bba8cf744
--- /dev/null
+++ b/tests/files/custom_cni/values.yaml
@@ -0,0 +1,11 @@
+---
+
+# We disable hubble so that helm doesn't try to generate any certificate.
+# This is not needed to test network_plugin/custom_cni anyway.
+hubble:
+ enabled: false
+
+ipam:
+ operator:
+ # Set the appropriate pods subnet
+ clusterPoolIPv4PodCIDR: "{{ kube_pods_subnet }}"
diff --git a/tests/files/packet_debian11-custom-cni.yml b/tests/files/packet_debian11-custom-cni.yml
new file mode 100644
index 000000000..e64fc8ef8
--- /dev/null
+++ b/tests/files/packet_debian11-custom-cni.yml
@@ -0,0 +1,9 @@
+---
+# Instance settings
+cloud_image: debian-11
+mode: default
+
+# Kubespray settings
+kube_network_plugin: custom_cni
+custom_cni_manifests:
+ - tests/files/custom_cni/cilium.yaml
--
GitLab