From 35682b52281fc5498ac3bce851503728ee8fee12 Mon Sep 17 00:00:00 2001
From: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>
Date: Thu, 6 Aug 2020 09:14:55 +0200
Subject: [PATCH] Fix cilium strict kube proxy replacement in HA (#6473)
* Update the cilium svc proxy test to HA mode
Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>
* Fix cilium strict kube-proxy in HA
Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>
* Add a single global endpoint variable
Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>
* Add cilium docs about kube-proxy replacement
Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>
* Fix issues in docs
Signed-off-by: Arthur Outhenin-Chalandre <arthur@cri.epita.fr>
---
docs/cilium.md | 13 +++++++++++++
roles/kubespray-defaults/defaults/main.yaml | 6 ++++++
.../cilium/templates/cilium-deploy.yml.j2 | 6 ++++++
.../cilium/templates/cilium-ds.yml.j2 | 6 ++++++
tests/files/packet_debian10-cilium-svc-proxy.yml | 2 +-
5 files changed, 32 insertions(+), 1 deletion(-)
create mode 100644 docs/cilium.md
diff --git a/docs/cilium.md b/docs/cilium.md
new file mode 100644
index 000000000..ea4294a94
--- /dev/null
+++ b/docs/cilium.md
@@ -0,0 +1,13 @@
+# Cilium
+
+## Kube-proxy replacement with Cilium
+
+Cilium can run without kube-proxy by setting `cilium_kube_proxy_replacement`
+to `strict`.
+
+Without kube-proxy, cilium needs to know the address of the kube-apiserver
+and this must be set globally for all cilium components (agents and operators).
+Hence, in this configuration in Kubespray, Cilium will always contact
+the external loadbalancer (even from a node in the control plane)
+and if there is no external load balancer It will ignore any local load
+balancer deployed by Kubespray and **only contacts the first master**.
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 5b7e5cd12..e30575f0c 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -448,6 +448,12 @@ loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}"
loadbalancer_apiserver_type: "nginx"
# applied if only external loadbalancer_apiserver is defined, otherwise ignored
apiserver_loadbalancer_domain_name: "lb-apiserver.kubernetes.local"
+kube_apiserver_global_endpoint: |-
+ {% if loadbalancer_apiserver is defined -%}
+ https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
+ {%- else -%}
+ https://{{ first_kube_master }}:{{ kube_apiserver_port }}
+ {%- endif %}
kube_apiserver_endpoint: |-
{% if loadbalancer_apiserver is defined -%}
https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }}
diff --git a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2 b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2
index 18fdad7bc..bf89286e2 100644
--- a/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-deploy.yml.j2
@@ -84,6 +84,12 @@ spec:
key: AWS_DEFAULT_REGION
name: cilium-aws
optional: true
+{% if cilium_kube_proxy_replacement == 'strict' %}
+ - name: KUBERNETES_SERVICE_HOST
+ value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+ - name: KUBERNETES_SERVICE_PORT
+ value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
image: "{{ cilium_operator_image_repo }}:{{ cilium_operator_image_tag }}"
imagePullPolicy: {{ k8s_image_pull_policy }}
name: cilium-operator
diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
index 9393c6f37..dbf26ab0f 100755
--- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
+++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2
@@ -43,6 +43,12 @@ spec:
fieldPath: metadata.namespace
- name: CILIUM_CLUSTERMESH_CONFIG
value: /var/lib/cilium/clustermesh/
+{% if cilium_kube_proxy_replacement == 'strict' %}
+ - name: KUBERNETES_SERVICE_HOST
+ value: "{{ kube_apiserver_global_endpoint | urlsplit('hostname') }}"
+ - name: KUBERNETES_SERVICE_PORT
+ value: "{{ kube_apiserver_global_endpoint | urlsplit('port') }}"
+{% endif %}
image: "{{cilium_image_repo}}:{{cilium_image_tag}}"
imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
diff --git a/tests/files/packet_debian10-cilium-svc-proxy.yml b/tests/files/packet_debian10-cilium-svc-proxy.yml
index eb163332c..9565a6ab6 100644
--- a/tests/files/packet_debian10-cilium-svc-proxy.yml
+++ b/tests/files/packet_debian10-cilium-svc-proxy.yml
@@ -1,7 +1,7 @@
---
# Instance settings
cloud_image: debian-10
-mode: separate
+mode: ha
# Kubespray settings
kube_network_plugin: cilium
--
GitLab