From 58f48500b1adac3f18466fa1c5cf8aa9d9838150 Mon Sep 17 00:00:00 2001
From: Florian Ruynat <florian234@hotmail.com>
Date: Wed, 15 Apr 2020 08:48:02 +0200
Subject: [PATCH] Update Flannel manifests, install script and version (0.12) +
fix tests scripts (#5937)
* Add CI_TEST_VARS to tests
* Update flannel to 0.12.0 (with new manifests) and disable tx/rx
offloading in networking test
---
README.md | 2 +-
docs/flannel.md | 6 ++
roles/download/defaults/main.yml | 2 +-
.../network_plugin/flannel/defaults/main.yml | 3 -
roles/network_plugin/flannel/tasks/main.yml | 16 ++++
.../flannel/templates/cni-flannel-rbac.yml.j2 | 65 +++++++++++---
.../flannel/templates/cni-flannel.yml.j2 | 88 +++++++++----------
tests/files/packet_centos7-flannel-addons.yml | 3 -
tests/scripts/testcases_run.sh | 10 +--
tests/testcases/040_check-network-adv.yml | 6 ++
10 files changed, 134 insertions(+), 67 deletions(-)
diff --git a/README.md b/README.md
index 57f4b1139..97cf78593 100644
--- a/README.md
+++ b/README.md
@@ -125,7 +125,7 @@ Note: Upstart/SysV init based OS types are not supported.
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- [cilium](https://github.com/cilium/cilium) v1.5.5
- [contiv](https://github.com/contiv/install) v1.2.1
- - [flanneld](https://github.com/coreos/flannel) v0.11.0
+ - [flanneld](https://github.com/coreos/flannel) v0.12.0
- [kube-router](https://github.com/cloudnativelabs/kube-router) v0.4.0
- [multus](https://github.com/intel/multus-cni) v3.4.1
- [weave](https://github.com/weaveworks/weave) v2.5.2
diff --git a/docs/flannel.md b/docs/flannel.md
index bea14d79c..d4a9b9713 100644
--- a/docs/flannel.md
+++ b/docs/flannel.md
@@ -1,5 +1,11 @@
# Flannel
+Flannel is a network fabric for containers, designed for Kubernetes
+
+**Warning:** You may encounter this [bug](https://github.com/coreos/flannel/pull/1282) with `VXLAN` backend, while waiting on a newer Flannel version the current workaround (`ethtool --offload flannel.1 rx off tx off`) is showcase in kubespray [networking test](tests/testcases/040_check-network-adv.yml:31).
+
+## Verifying flannel install
+
* Flannel configuration file should have been created there
```ShellSession
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index 5d5cf8a60..f78fb662e 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -72,7 +72,7 @@ calico_policy_version: "v3.13.2"
calico_typha_version: "v3.13.2"
typha_enabled: false
-flannel_version: "v0.11.0"
+flannel_version: "v0.12.0"
flannel_cni_version: "v0.3.0"
cni_version: "v0.8.5"
diff --git a/roles/network_plugin/flannel/defaults/main.yml b/roles/network_plugin/flannel/defaults/main.yml
index f7f773fdc..09b18fd3b 100644
--- a/roles/network_plugin/flannel/defaults/main.yml
+++ b/roles/network_plugin/flannel/defaults/main.yml
@@ -23,6 +23,3 @@ flannel_memory_limit: 500M
flannel_cpu_limit: 300m
flannel_memory_requests: 64M
flannel_cpu_requests: 150m
-
-# Legacy directory, will be removed if found.
-flannel_cert_dir: /etc/flannel/certs
diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml
index d5a725baf..d187a8e19 100644
--- a/roles/network_plugin/flannel/tasks/main.yml
+++ b/roles/network_plugin/flannel/tasks/main.yml
@@ -9,3 +9,19 @@
register: flannel_node_manifests
when:
- inventory_hostname in groups['kube-master']
+
+- name: Flannel | Set CNI directory permissions
+ file:
+ path: /opt/cni/bin
+ state: directory
+ owner: kube
+ recurse: true
+ mode: 0755
+ register: cni_bin_dir
+
+- name: Flannel | Copy CNI plugins
+ unarchive:
+ src: "{{ local_release_dir }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
+ dest: "/opt/cni/bin"
+ mode: 0755
+ remote_src: yes
diff --git a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2
index ceb2c09e0..ce4980ccb 100644
--- a/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2
+++ b/roles/network_plugin/flannel/templates/cni-flannel-rbac.yml.j2
@@ -3,13 +3,66 @@ apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
- namespace: "kube-system"
+ namespace: kube-system
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+ name: psp.flannel.unprivileged
+ annotations:
+ seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
+ seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
+{% if podsecuritypolicy_enabled and apparmor_enabled %}
+ apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
+ apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
+{% endif %}
+spec:
+ privileged: false
+ volumes:
+ - configMap
+ - secret
+ - emptyDir
+ - hostPath
+ allowedHostPaths:
+ - pathPrefix: "/etc/cni/net.d"
+ - pathPrefix: "/etc/kube-flannel"
+ - pathPrefix: "/run/flannel"
+ readOnlyRootFilesystem: false
+ # Users and groups
+ runAsUser:
+ rule: RunAsAny
+ supplementalGroups:
+ rule: RunAsAny
+ fsGroup:
+ rule: RunAsAny
+ # Privilege Escalation
+ allowPrivilegeEscalation: false
+ defaultAllowPrivilegeEscalation: false
+ # Capabilities
+ allowedCapabilities: ['NET_ADMIN']
+ defaultAddCapabilities: []
+ requiredDropCapabilities: []
+ # Host namespaces
+ hostPID: false
+ hostIPC: false
+ hostNetwork: true
+ hostPorts:
+ - min: 0
+ max: 65535
+ # SELinux
+ seLinux:
+ # SELinux is unused in CaaSP
+ rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: flannel
rules:
+ - apiGroups: ['extensions']
+ resources: ['podsecuritypolicies']
+ verbs: ['use']
+ resourceNames: ['psp.flannel.unprivileged']
- apiGroups:
- ""
resources:
@@ -29,14 +82,6 @@ rules:
- nodes/status
verbs:
- patch
- - apiGroups:
- - policy
- resourceNames:
- - privileged
- resources:
- - podsecuritypolicies
- verbs:
- - use
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
@@ -49,4 +94,4 @@ roleRef:
subjects:
- kind: ServiceAccount
name: flannel
- namespace: "kube-system"
\ No newline at end of file
+ namespace: kube-system
diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
index 9afabd151..135be5c94 100644
--- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
+++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2
@@ -3,28 +3,27 @@ kind: ConfigMap
apiVersion: v1
metadata:
name: kube-flannel-cfg
- namespace: "kube-system"
+ namespace: kube-system
labels:
tier: node
app: flannel
data:
cni-conf.json: |
{
- "name":"cni0",
- "cniVersion":"0.3.1",
- "plugins":[
+ "name": "cbr0",
+ "cniVersion": "0.3.1",
+ "plugins": [
{
- "type":"flannel",
- "delegate":{
- "forceAddress":true,
+ "type": "flannel",
+ "delegate": {
"hairpinMode": true,
- "isDefaultGateway":true
+ "isDefaultGateway": true
}
},
{
- "type":"portmap",
- "capabilities":{
- "portMappings":true
+ "type": "portmap",
+ "capabilities": {
+ "portMappings": true
}
}
]
@@ -41,25 +40,22 @@ apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel
- namespace: "kube-system"
+ namespace: kube-system
labels:
tier: node
- k8s-app: flannel
+ app: flannel
spec:
selector:
matchLabels:
- tier: node
- k8s-app: flannel
+ app: flannel
template:
metadata:
labels:
tier: node
- k8s-app: flannel
+ app: flannel
spec:
priorityClassName: system-node-critical
serviceAccountName: flannel
- nodeSelector:
- beta.kubernetes.io/os: linux
containers:
- name: kube-flannel
image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
@@ -73,7 +69,9 @@ spec:
memory: {{ flannel_memory_requests }}
command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"{% if flannel_interface is defined %}, "--iface={{ flannel_interface }}"{% endif %}{% if flannel_interface_regexp is defined %}, "--iface-regex={{ flannel_interface_regexp }}"{% endif %} ]
securityContext:
- privileged: true
+ privileged: false
+ capabilities:
+ add: ["NET_ADMIN"]
env:
- name: POD_NAME
valueFrom:
@@ -83,45 +81,47 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- - name: POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
volumeMounts:
- name: run
- mountPath: /run
- - name: cni
- mountPath: /etc/cni/net.d
+ mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: beta.kubernetes.io/os
+ operator: In
+ values:
+ - linux
+ - key: beta.kubernetes.io/arch
+ operator: In
+ values:
+ - amd64
+ initContainers:
- name: install-cni
image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }}
- command: ["/install-cni.sh"]
- env:
- # The CNI network config to install on each node.
- - name: CNI_NETWORK_CONFIG
- valueFrom:
- configMapKeyRef:
- name: kube-flannel-cfg
- key: cni-conf.json
- - name: CNI_CONF_NAME
- value: "10-flannel.conflist"
+ command:
+ - cp
+ args:
+ - -f
+ - /etc/kube-flannel/cni-conf.json
+ - /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
- mountPath: /host/etc/cni/net.d
- - name: host-cni-bin
- mountPath: /host/opt/cni/bin/
+ mountPath: /etc/cni/net.d
+ - name: flannel-cfg
+ mountPath: /etc/kube-flannel/
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
tolerations:
- - operator: Exists
- # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12)
- - key: CriticalAddonsOnly
- operator: "Exists"
+ - operator: Exists
+ effect: NoSchedule
volumes:
- name: run
hostPath:
- path: /run
+ path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
diff --git a/tests/files/packet_centos7-flannel-addons.yml b/tests/files/packet_centos7-flannel-addons.yml
index 60ccb69c0..a90e5eaf5 100644
--- a/tests/files/packet_centos7-flannel-addons.yml
+++ b/tests/files/packet_centos7-flannel-addons.yml
@@ -28,6 +28,3 @@ enable_nodelocaldns: false
kube_oidc_url: https://accounts.google.com/.well-known/openid-configuration
kube_oidc_client_id: kubespray-example
-
-# Temp set k8s ver to 1.16.8
-kube_version: v1.16.8
\ No newline at end of file
diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh
index 85d28c2f1..e406efc3f 100755
--- a/tests/scripts/testcases_run.sh
+++ b/tests/scripts/testcases_run.sh
@@ -52,19 +52,19 @@ fi
# Tests Cases
## Test Master API
-ansible-playbook --limit "all:!fake_hosts" tests/testcases/010_check-apiserver.yml $ANSIBLE_LOG_LEVEL
+ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/010_check-apiserver.yml $ANSIBLE_LOG_LEVEL
## Test that all pods are Running
-ansible-playbook --limit "all:!fake_hosts" tests/testcases/015_check-pods-running.yml $ANSIBLE_LOG_LEVEL
+ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/015_check-pods-running.yml $ANSIBLE_LOG_LEVEL
## Test that all nodes are Ready
-ansible-playbook --limit "all:!fake_hosts" tests/testcases/020_check-nodes-ready.yml $ANSIBLE_LOG_LEVEL
+ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/020_check-nodes-ready.yml $ANSIBLE_LOG_LEVEL
## Test pod creation and ping between them
-ansible-playbook --limit "all:!fake_hosts" tests/testcases/030_check-network.yml $ANSIBLE_LOG_LEVEL
+ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/030_check-network.yml $ANSIBLE_LOG_LEVEL
## Advanced DNS checks
-ansible-playbook --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml $ANSIBLE_LOG_LEVEL
+ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} tests/testcases/040_check-network-adv.yml $ANSIBLE_LOG_LEVEL
## Kubernetes conformance tests
ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_hosts" tests/testcases/100_check-k8s-conformance.yml $ANSIBLE_LOG_LEVEL
diff --git a/tests/testcases/040_check-network-adv.yml b/tests/testcases/040_check-network-adv.yml
index de2197e09..9818ef38d 100644
--- a/tests/testcases/040_check-network-adv.yml
+++ b/tests/testcases/040_check-network-adv.yml
@@ -14,6 +14,12 @@
netchecker_port: 31081
tasks:
+ - name: Flannel | Disable tx and rx offloading on VXLAN interfaces (see https://github.com/coreos/flannel/pull/1282)
+ shell: "ethtool --offload flannel.1 rx off tx off"
+ ignore_errors: true
+ when:
+ - kube_network_plugin == 'flannel'
+
- name: Force binaries directory for Container Linux by CoreOS and Flatcar
set_fact:
bin_dir: "/opt/bin"
--
GitLab