From 307f598bc88bb71d2e290e47cdfed90dbe19ab7c Mon Sep 17 00:00:00 2001
From: Florian Ruynat <16313165+floryut@users.noreply.github.com>
Date: Fri, 22 Jul 2022 15:28:07 +0200
Subject: [PATCH] Move flannel to etcd datastore
---
roles/network_plugin/canal/tasks/main.yml | 23 ++-
.../canal-calico-kube-controllers.yml.j2 | 43 ++++-
.../canal/templates/canal-config.yaml.j2 | 71 +++----
.../canal/templates/canal-cr-flannel.yml.j2 | 2 +-
.../canal/templates/canal-cr.yml.j2 | 30 +++
.../canal/templates/canal-crb-canal.yml.j2 | 12 ++
.../canal/templates/canal-node-sa.yml.j2 | 6 +
.../canal/templates/canal-node.yaml.j2 | 177 ++++++++++++++----
.../templates/canal-secret-calico-etcd.yml.j2 | 18 ++
.../canal/templates/cni-canal.conflist.j2 | 59 +++---
10 files changed, 336 insertions(+), 105 deletions(-)
create mode 100644 roles/network_plugin/canal/templates/canal-cr.yml.j2
create mode 100644 roles/network_plugin/canal/templates/canal-crb-canal.yml.j2
create mode 100644 roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2
diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml
index 6ff54714e..4117d1ca3 100644
--- a/roles/network_plugin/canal/tasks/main.yml
+++ b/roles/network_plugin/canal/tasks/main.yml
@@ -28,10 +28,28 @@
- {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
- {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
+- name: Slurp etcd cacert file
+ slurp:
+ src: "{{ canal_cert_dir }}/ca_cert.crt"
+ register: etcd_ca_cert_file
+ failed_when: false
+
+- name: Slurp etcd cert file
+ slurp:
+ src: "{{ canal_cert_dir }}/cert.crt"
+ register: etcd_cert_file
+ failed_when: false
+
+- name: Slurp etcd key file
+ slurp:
+ src: "{{ canal_cert_dir }}/key.pem"
+ register: etcd_key_file
+ failed_when: false
+
# Flannel need etcd v2 API
- name: Canal | Set Flannel etcd configuration
command: |-
- {{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config \
+ {{ bin_dir }}/etcdctl set /coreos.com/network/config \
'{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
register: output
retries: 4
@@ -53,14 +71,17 @@
dest: "{{ kube_config_dir }}/{{ item.file }}"
mode: 0644
with_items:
+ - {name: canal-calico-etcd-secret, file: canal-secret-calico-etcd.yml, type: secret}
- {name: canal-config, file: canal-config.yaml, type: cm}
- {name: canal-node, file: canal-node.yaml, type: ds}
- {name: canal-kube-controllers, file: canal-calico-kube-controllers.yml, type: deployment}
+ - {name: canal-cr, file: canal-cr.yml, type: clusterrole}
- {name: canal, file: canal-node-sa.yml, type: sa}
- {name: calico-cr, file: canal-cr-calico-node.yml, type: clusterrole}
- {name: calico-kube-cr, file: canal-cr-calico-kube-controllers.yml, type: clusterrole}
- {name: calico-crd, file: canal-crd-calico.yml, type: crd}
- {name: flannel, file: canal-cr-flannel.yml, type: clusterrole}
+ - {name: canal, file: canal-crb-canal.yml, type: clusterrolebinding}
- {name: canal-calico, file: canal-crb-calico.yml, type: clusterrolebinding}
- {name: canal-flannel, file: canal-crb-flannel.yml, type: clusterrolebinding}
register: canal_manifests
diff --git a/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2 b/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2
index 6ce75068d..1417022a8 100644
--- a/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2
+++ b/roles/network_plugin/canal/templates/canal-calico-kube-controllers.yml.j2
@@ -33,15 +33,45 @@ spec:
effect: NoSchedule
serviceAccountName: calico-kube-controllers
priorityClassName: system-cluster-critical
+ # The controllers must run in the host network namespace so that
+ # it isn't governed by policy that would prevent it from working.
+ hostNetwork: true
containers:
- name: calico-kube-controllers
image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+ imagePullPolicy: {{ k8s_image_pull_policy }}
env:
+ # The location of the etcd cluster.
+ - name: ETCD_ENDPOINTS
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_endpoints
+ # Location of the CA certificate for etcd.
+ - name: ETCD_CA_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_ca
+ # Location of the client key for etcd.
+ - name: ETCD_KEY_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_key
+ # Location of the client certificate for etcd.
+ - name: ETCD_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_cert
# Choose which controllers to run.
- name: ENABLED_CONTROLLERS
- value: node
- - name: DATASTORE_TYPE
- value: kubernetes
+ value: policy,namespace,serviceaccount,workloadendpoint,node
+ volumeMounts:
+ # Mount in the etcd TLS secrets.
+ - mountPath: /calico-secrets
+ name: etcd-certs
livenessProbe:
exec:
command:
@@ -57,3 +87,10 @@ spec:
- /usr/bin/check-status
- -r
periodSeconds: 10
+ volumes:
+ # Mount in the etcd TLS secrets with mode 400.
+ # See https://kubernetes.io/docs/concepts/configuration/secret/
+ - name: etcd-certs
+ secret:
+ secretName: calico-etcd-secrets
+ defaultMode: 0440
diff --git a/roles/network_plugin/canal/templates/canal-config.yaml.j2 b/roles/network_plugin/canal/templates/canal-config.yaml.j2
index 9bac59dca..8aab6fb75 100644
--- a/roles/network_plugin/canal/templates/canal-config.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-config.yaml.j2
@@ -7,6 +7,14 @@ metadata:
name: canal-config
namespace: kube-system
data:
+ # Configure this with the location of your etcd cluster.
+ etcd_endpoints: "{{ etcd_access_addresses }}"
+ # If you're using TLS enabled etcd uncomment the following.
+ # You must also populate the Secret below with these files.
+ etcd_ca: "/calico-secrets/etcd-ca"
+ etcd_cert: "/calico-secrets/etcd-cert"
+ etcd_key: "/calico-secrets/etcd-key"
+
# Typha is disabled.
typha_service_name: "none"
@@ -28,41 +36,39 @@ data:
# values in this config will be automatically populated.
cni_network_config: |-
{
- "name": "k8s-pod-network",
- "cniVersion": "0.3.1",
- "plugins": [
- {
- "type": "calico",
- "log_level": "info",
+ "name": "canal",
+ "cniVersion": "0.3.1",
+ "plugins": [
+ {
+ "type": "flannel",
+ "delegate": {
+ "type": "calico",
+ "include_default_routes": true,
+ "etcd_endpoints": "__ETCD_ENDPOINTS__",
+ "etcd_key_file": "__ETCD_KEY_FILE__",
+ "etcd_cert_file": "__ETCD_CERT_FILE__",
+ "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
+ "log_level": "info",
{% if calico_cni_log_file_path %}
- "log_file_path": "{{ calico_cni_log_file_path }}",
+ "log_file_path": "{{ calico_cni_log_file_path }}",
{% endif %}
- "datastore_type": "kubernetes",
- "nodename": "__KUBERNETES_NODE_NAME__",
- "mtu": __CNI_MTU__,
- "ipam": {
- "type": "host-local",
- "subnet": "usePodCidr"
- },
- "policy": {
- "type": "k8s"
- },
- "kubernetes": {
- "kubeconfig": "__KUBECONFIG_FILEPATH__"
- }
- },
- {
- "type": "portmap",
- "snat": true,
- "capabilities": {"portMappings": true}
- },
- {
- "type": "bandwidth",
- "capabilities": {"bandwidth": true}
- }
- ]
+ "policy": {
+ "type": "k8s",
+ "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+ "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+ },
+ "kubernetes": {
+ "kubeconfig": "__KUBECONFIG_FILEPATH__"
+ }
+ }
+ },
+ {
+ "type": "portmap",
+ "capabilities": {"portMappings": true},
+ "snat": true
+ }
+ ]
}
-
# Flannel network configuration. Mounted into the flannel container.
net-conf.json: |
{
@@ -71,3 +77,4 @@ data:
"Type": "vxlan"
}
}
+
diff --git a/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2 b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
index 9cb8e7ff8..b2236d161 100644
--- a/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
+++ b/roles/network_plugin/canal/templates/canal-cr-flannel.yml.j2
@@ -1,4 +1,4 @@
----
+# Flannel ClusterRole
# Pulled from https://github.com/coreos/flannel/blob/master/Documentation/kube-flannel-rbac.yml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
diff --git a/roles/network_plugin/canal/templates/canal-cr.yml.j2 b/roles/network_plugin/canal/templates/canal-cr.yml.j2
new file mode 100644
index 000000000..1209c7b1a
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-cr.yml.j2
@@ -0,0 +1,30 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: canal
+rules:
+ # Used for creating service account tokens to be used by the CNI plugin
+ - apiGroups: [""]
+ resources:
+ - serviceaccounts/token
+ verbs:
+ - create
+ - apiGroups: [""]
+ resources:
+ - pods
+ - nodes
+ - namespaces
+ verbs:
+ - get
+ # Pod CIDR auto-detection on kubeadm needs access to config maps.
+ - apiGroups: [""]
+ resources:
+ - configmaps
+ verbs:
+ - get
+ - apiGroups:
+ - ""
+ resources:
+ - nodes
+ verbs:
+ - list
diff --git a/roles/network_plugin/canal/templates/canal-crb-canal.yml.j2 b/roles/network_plugin/canal/templates/canal-crb-canal.yml.j2
new file mode 100644
index 000000000..9fcb0fcf6
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-crb-canal.yml.j2
@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+ name: canal
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: canal
+subjects:
+- kind: ServiceAccount
+ name: canal
+ namespace: kube-system
diff --git a/roles/network_plugin/canal/templates/canal-node-sa.yml.j2 b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2
index 582d55bbd..954f6d78e 100644
--- a/roles/network_plugin/canal/templates/canal-node-sa.yml.j2
+++ b/roles/network_plugin/canal/templates/canal-node-sa.yml.j2
@@ -4,3 +4,9 @@ kind: ServiceAccount
metadata:
name: canal
namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: calico-kube-controllers
+ namespace: kube-system
diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2
index 737be705e..529d4b967 100644
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@@ -44,6 +44,7 @@ spec:
# and CNI network config file on each node.
- name: install-cni
image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }}
+ imagePullPolicy: {{ k8s_image_pull_policy }}
command: ["/opt/cni/bin/install"]
envFrom:
- configMapRef:
@@ -71,6 +72,30 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
+ # The location of the etcd cluster.
+ - name: ETCD_ENDPOINTS
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_endpoints
+ # Location of the CA certificate for etcd.
+ - name: ETCD_CA_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_ca
+ # Location of the client key for etcd.
+ - name: ETCD_KEY_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_key
+ # Location of the client certificate for etcd.
+ - name: ETCD_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_cert
# CNI MTU Config variable
- name: CNI_MTU
valueFrom:
@@ -85,6 +110,8 @@ spec:
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
+ - mountPath: /calico-secrets
+ name: etcd-certs
securityContext:
privileged: true
# This init container mounts the necessary filesystems needed by the BPF data plane
@@ -125,17 +152,32 @@ spec:
name: kubernetes-services-endpoint
optional: true
env:
- # Use Kubernetes API as the backing datastore.
- - name: DATASTORE_TYPE
- value: "kubernetes"
- # Configure route aggregation based on pod CIDR.
- - name: USE_POD_CIDR
- value: "true"
- # Wait for the datastore.
- - name: WAIT_FOR_DATASTORE
- value: "true"
- # Set based on the k8s node name.
- - name: NODENAME
+ # The location of the etcd cluster.
+ - name: ETCD_ENDPOINTS
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_endpoints
+ # Location of the CA certificate for etcd.
+ - name: ETCD_CA_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_ca
+ # Location of the client key for etcd.
+ - name: ETCD_KEY_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_key
+ # Location of the client certificate for etcd.
+ - name: ETCD_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_cert
+ # Set noderef for node controller.
+ - name: CALICO_K8S_NODE_REF
valueFrom:
fieldRef:
fieldPath: spec.nodeName
@@ -221,6 +263,8 @@ spec:
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
+ - mountPath: /calico-secrets
+ name: etcd-certs
- name: policysync
mountPath: /var/run/nodeagent
# For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
@@ -230,48 +274,94 @@ spec:
- name: cni-log-dir
mountPath: /var/log/calico/cni
readOnly: true
- # This container runs flannel using the kube-subnet-mgr backend
- # for allocating subnets.
- - name: kube-flannel
+ # Runs the flannel daemon to enable vxlan networking between
+ # container hosts.
+ - name: flannel
image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
+ command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"]
imagePullPolicy: {{ k8s_image_pull_policy }}
- command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
- securityContext:
- privileged: true
- resources:
- limits:
- cpu: {{ flannel_cpu_limit }}
- memory: {{ flannel_memory_limit }}
- requests:
- cpu: {{ flannel_cpu_requests }}
- memory: {{ flannel_memory_requests }}
env:
- - name: POD_NAME
+ # The location of the etcd cluster.
+ - name: FLANNELD_ETCD_ENDPOINTS
valueFrom:
- fieldRef:
- fieldPath: metadata.name
- - name: POD_NAMESPACE
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_endpoints
+ # Location of the CA certificate for etcd.
+ - name: ETCD_CA_CERT_FILE
valueFrom:
- fieldRef:
- fieldPath: metadata.namespace
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_ca
+ # Location of the client key for etcd.
+ - name: ETCD_KEY_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_key
+ # Location of the client certificate for etcd.
+ - name: ETCD_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_cert
+ # Location of the CA certificate for etcd.
+ - name: FLANNELD_ETCD_CAFILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_ca
+ # Location of the client key for etcd.
+ - name: FLANNELD_ETCD_KEYFILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_key
+ # Location of the client certificate for etcd.
+ - name: FLANNELD_ETCD_CERTFILE
+ valueFrom:
+ configMapKeyRef:
+ name: canal-config
+ key: etcd_cert
+ # The interface flannel should run on.
- name: FLANNELD_IFACE
valueFrom:
configMapKeyRef:
name: canal-config
key: canal_iface
+ # Perform masquerade on traffic leaving the pod cidr.
- name: FLANNELD_IP_MASQ
valueFrom:
configMapKeyRef:
name: canal-config
key: masquerade
+ # Write the subnet.env file to the mounted directory.
+ - name: FLANNELD_SUBNET_FILE
+ value: "/run/flannel/subnet.env"
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ securityContext:
+ privileged: true
volumeMounts:
- - mountPath: /run/xtables.lock
- name: xtables-lock
- readOnly: false
- - name: flannel-cfg
- mountPath: /etc/kube-flannel/
+ - mountPath: /etc/resolv.conf
+ name: resolv
+ - mountPath: /run/flannel
+ name: run-flannel
+ - mountPath: /calico-secrets
+ name: etcd-certs
+ - name: flannel-cfg
+ mountPath: /etc/kube-flannel/
volumes:
- # Used by canal.
+ - name: flannel-cfg
+ configMap:
+ name: canal-config
+ # Used by canal-node.
- name: lib-modules
hostPath:
path: /lib/modules
@@ -298,9 +388,12 @@ spec:
hostPath:
path: /proc
# Used by flannel.
- - name: flannel-cfg
- configMap:
- name: canal-config
+ - name: run-flannel
+ hostPath:
+ path: /run/flannel
+ - name: resolv
+ hostPath:
+ path: /etc/resolv.conf
# Used to install CNI.
- name: cni-bin-dir
hostPath:
@@ -312,6 +405,12 @@ spec:
- name: cni-log-dir
hostPath:
path: /var/log/calico/cni
+ # Mount in the etcd TLS secrets with mode 400.
+ # See https://kubernetes.io/docs/concepts/configuration/secret/
+ - name: etcd-certs
+ secret:
+ secretName: calico-etcd-secrets
+ defaultMode: 0400
# Used to create per-pod Unix Domain Sockets
- name: policysync
hostPath:
diff --git a/roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2 b/roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2
new file mode 100644
index 000000000..bed51c713
--- /dev/null
+++ b/roles/network_plugin/canal/templates/canal-secret-calico-etcd.yml.j2
@@ -0,0 +1,18 @@
+# Source: calico/templates/calico-etcd-secrets.yaml
+# The following contains k8s Secrets for use with a TLS enabled etcd cluster.
+# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+ name: calico-etcd-secrets
+ namespace: kube-system
+data:
+ # Populate the following with etcd TLS configuration if desired, but leave blank if
+ # not using TLS for etcd.
+ # The keys below should be uncommented and the values populated with the base64
+ # encoded contents of each file that would be associated with the TLS data.
+ # Example command for encoding a file contents: cat <file> | base64 -w 0
+ etcd-key: {{ etcd_key_file.content }}
+ etcd-cert: {{ etcd_cert_file.content }}
+ etcd-ca: {{ etcd_ca_cert_file.content }}
diff --git a/roles/network_plugin/canal/templates/cni-canal.conflist.j2 b/roles/network_plugin/canal/templates/cni-canal.conflist.j2
index 03619b246..3902a81c0 100644
--- a/roles/network_plugin/canal/templates/cni-canal.conflist.j2
+++ b/roles/network_plugin/canal/templates/cni-canal.conflist.j2
@@ -1,33 +1,34 @@
-{
- "name": "cni0",
- "cniVersion":"0.3.1",
- "plugins":[
{
- "type": "flannel",
- "delegate": {
- "type": "calico",
- "include_default_routes": true,
- "etcd_endpoints": "{{ etcd_access_addresses }}",
- "etcd_key_file": "{{ canal_cert_dir }}/key.pem",
- "etcd_cert_file": "{{ canal_cert_dir }}/cert.crt",
- "etcd_ca_cert_file": "{{ canal_cert_dir }}/ca_cert.crt",
- "log_level": "info",
+ "name": "canal",
+ "cniVersion": "0.3.1",
+ "plugins": [
+ {
+ "type": "flannel",
+ "delegate": {
+ "type": "calico",
+ "include_default_routes": true,
+ "etcd_endpoints": "__ETCD_ENDPOINTS__",
+ "etcd_key_file": "__ETCD_KEY_FILE__",
+ "etcd_cert_file": "__ETCD_CERT_FILE__",
+ "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
+ "log_level": "info",
{% if calico_cni_log_file_path %}
- "log_file_path": "{{ calico_cni_log_file_path }}",
+ "log_file_path": "{{ calico_cni_log_file_path }}",
{% endif %}
- "policy": {
- "type": "k8s"
- },
- "kubernetes": {
- "kubeconfig": "__KUBECONFIG_FILEPATH__"
- }
- }
- },
- {
- "type":"portmap",
- "capabilities":{
- "portMappings":true
- }
+ "policy": {
+ "type": "k8s",
+ "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+ "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+ },
+ "kubernetes": {
+ "kubeconfig": "__KUBECONFIG_FILEPATH__"
+ }
+ }
+ },
+ {
+ "type": "portmap",
+ "capabilities": {"portMappings": true},
+ "snat": true
+ }
+ ]
}
- ]
-}
--
GitLab