From c929b5e82e46465f73febb222f24da16de58e845 Mon Sep 17 00:00:00 2001
From: Florian Ruynat <florian234@hotmail.com>
Date: Wed, 15 Apr 2020 12:10:03 +0200
Subject: [PATCH] Upgrade kube-ovn to v1.1.0 and move test from centos7 to
centos8 (#5852)
---
.gitlab-ci/packet.yml | 2 +-
docs/kube-ovn.md | 6 +
roles/download/defaults/main.yml | 48 +----
.../network_plugin/kube-ovn/defaults/main.yml | 7 +-
.../templates/cni-kube-ovn-crd.yml.j2 | 19 +-
.../kube-ovn/templates/cni-kube-ovn.yml.j2 | 195 ++++++++++++++++--
.../kube-ovn/templates/cni-ovn.yml.j2 | 71 +++++--
...be-ovn.yml => packet_centos8-kube-ovn.yml} | 7 +-
8 files changed, 271 insertions(+), 84 deletions(-)
rename tests/files/{packet_centos7-kube-ovn.yml => packet_centos8-kube-ovn.yml} (55%)
diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml
index 3f83db566..ca1da70fc 100644
--- a/.gitlab-ci/packet.yml
+++ b/.gitlab-ci/packet.yml
@@ -108,7 +108,7 @@ packet_centos7-calico-ha-once-localhost:
services:
- docker:18.09.9-dind
-packet_centos7-kube-ovn:
+packet_centos8-kube-ovn:
stage: deploy-part2
extends: .packet
when: on_success
diff --git a/docs/kube-ovn.md b/docs/kube-ovn.md
index d811b13d1..375c7a4d5 100644
--- a/docs/kube-ovn.md
+++ b/docs/kube-ovn.md
@@ -4,6 +4,12 @@ Kube-OVN integrates the OVN-based Network Virtualization with Kubernetes. It off
For more information please check [Kube-OVN documentation](https://github.com/alauda/kube-ovn)
+**Warning:** Kernel version (`cat /proc/version`) needs to be different than `3.10.0-862` or kube-ovn won't start and will print this message:
+
+```bash
+kernel version 3.10.0-862 has a nat related bug that will affect ovs function, please update to a version greater than 3.10.0-898
+```
+
## How to use it
Enable kube-ovn in `group_vars/k8s-cluster/k8s-cluster.yml`
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index f78fb662e..202378bec 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -63,6 +63,9 @@ docker_image_repo: "docker.io"
# quay image repo define
quay_image_repo: "quay.io"
+# alauda.cn image repo (for kube-ovn...)
+alauda_image_repo: "index.alauda.cn"
+
# TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
# after migration to container download
calico_version: "v3.13.2"
@@ -81,7 +84,7 @@ weave_version: 2.6.2
pod_infra_version: 3.1
contiv_version: 1.2.1
cilium_version: "v1.7.2"
-kube_ovn_version: "v0.6.0"
+kube_ovn_version: "v1.1.0"
kube_router_version: "v0.4.0"
multus_version: "v3.4.1"
@@ -472,14 +475,8 @@ cilium_init_image_repo: "{{ docker_image_repo }}/cilium/cilium-init"
cilium_init_image_tag: "2019-04-05"
cilium_operator_image_repo: "{{ docker_image_repo }}/cilium/operator"
cilium_operator_image_tag: "{{ cilium_version }}"
-kube_ovn_db_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-db"
-kube_ovn_node_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-node"
-kube_ovn_cni_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-cni"
-kube_ovn_controller_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn-controller"
-kube_ovn_db_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_node_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_controller_image_tag: "{{ kube_ovn_version }}"
-kube_ovn_cni_image_tag: "{{ kube_ovn_version }}"
+kube_ovn_container_image_repo: "{{ alauda_image_repo }}/alaudak8s/kube-ovn"
+kube_ovn_container_image_tag: "{{ kube_ovn_version }}"
kube_router_image_repo: "{{ docker_image_repo }}/cloudnativelabs/kube-router"
kube_router_image_tag: "{{ kube_router_version }}"
multus_image_repo: "{{ docker_image_repo }}/nfvpe/multus"
@@ -836,38 +833,11 @@ downloads:
groups:
- k8s-cluster
- kube_ovn_db:
- enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
- container: true
- repo: "{{ kube_ovn_db_image_repo }}"
- tag: "{{ kube_ovn_db_image_tag }}"
- sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
- groups:
- - k8s-cluster
-
- kube_ovn_node:
- enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
- container: true
- repo: "{{ kube_ovn_node_image_repo }}"
- tag: "{{ kube_ovn_node_image_tag }}"
- sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
- groups:
- - k8s-cluster
-
- kube_ovn_controller:
- enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
- container: true
- repo: "{{ kube_ovn_controller_image_repo }}"
- tag: "{{ kube_ovn_controller_image_tag }}"
- sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
- groups:
- - k8s-cluster
-
- kube_ovn_cni:
+ kube_ovn:
enabled: "{{ kube_network_plugin == 'kube-ovn' }}"
container: true
- repo: "{{ kube_ovn_cni_image_repo }}"
- tag: "{{ kube_ovn_cni_image_tag }}"
+ repo: "{{ kube_ovn_container_image_repo }}"
+ tag: "{{ kube_ovn_container_image_tag }}"
sha256: "{{ kube_ovn_digest_checksum|default(None) }}"
groups:
- k8s-cluster
diff --git a/roles/network_plugin/kube-ovn/defaults/main.yml b/roles/network_plugin/kube-ovn/defaults/main.yml
index fb0cfc63d..9cc4c5c84 100644
--- a/roles/network_plugin/kube-ovn/defaults/main.yml
+++ b/roles/network_plugin/kube-ovn/defaults/main.yml
@@ -7,5 +7,10 @@ kube_ovn_node_cpu_request: 100m
kube_ovn_node_memory_request: 300Mi
kube_ovn_node_cpu_limit: 200m
kube_ovn_node_memory_limit: 500Mi
+kube_ovn_pinger_cpu_request: 100m
+kube_ovn_pinger_memory_request: 300Mi
+kube_ovn_pinger_cpu_limit: 200m
+kube_ovn_pinger_memory_limit: 400Mi
-traffic_mirror: true
\ No newline at end of file
+traffic_mirror: true
+encap_checksum: true
\ No newline at end of file
diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
index 2f3c99f9b..2a16dcaf4 100644
--- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn-crd.yml.j2
@@ -40,7 +40,12 @@ spec:
kind: Subnet
shortNames:
- subnet
+ subresources:
+ status: {}
additionalPrinterColumns:
+ - name: Provider
+ type: string
+ JSONPath: .spec.provider
- name: Protocol
type: string
JSONPath: .spec.protocol
@@ -53,11 +58,23 @@ spec:
- name: NAT
type: boolean
JSONPath: .spec.natOutgoing
+ - name: Default
+ type: boolean
+ JSONPath: .spec.default
+ - name: GatewayType
+ type: string
+ JSONPath: .spec.gatewayType
+ - name: Used
+ type: integer
+ JSONPath: .status.usingIPs
+ - name: Available
+ type: integer
+ JSONPath: .status.availableIPs
validation:
openAPIV3Schema:
properties:
spec:
- required: ["cidrBlock","gateway"]
+ required: ["cidrBlock"]
properties:
cidrBlock:
type: "string"
diff --git a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
index ea7fcf93b..c67eaf500 100644
--- a/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-kube-ovn.yml.j2
@@ -34,11 +34,12 @@ spec:
matchLabels:
app: kube-ovn-controller
topologyKey: kubernetes.io/hostname
+ priorityClassName: system-cluster-critical
serviceAccountName: ovn
hostNetwork: true
containers:
- name: kube-ovn-controller
- image: {{ kube_ovn_controller_image_repo }}:{{ kube_ovn_controller_image_tag }}
+ image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
command:
- /kube-ovn/start-controller.sh
@@ -60,25 +61,19 @@ spec:
readinessProbe:
exec:
command:
- - nc
- - -z
- - -w3
- - 127.0.0.1
- - "10660"
+ - sh
+ - /kube-ovn/kube-ovn-controller-healthcheck.sh
periodSeconds: 3
livenessProbe:
exec:
command:
- - nc
- - -z
- - -w3
- - 127.0.0.1
- - "10660"
+ - sh
+ - /kube-ovn/kube-ovn-controller-healthcheck.sh
initialDelaySeconds: 30
periodSeconds: 7
failureThreshold: 5
nodeSelector:
- beta.kubernetes.io/os: "linux"
+ kubernetes.io/os: "linux"
---
kind: DaemonSet
@@ -94,7 +89,7 @@ spec:
matchLabels:
app: kube-ovn-cni
updateStrategy:
- type: RollingUpdate
+ type: OnDelete
template:
metadata:
labels:
@@ -105,14 +100,18 @@ spec:
tolerations:
- operator: Exists
effect: NoSchedule
+ priorityClassName: system-cluster-critical
serviceAccountName: ovn
hostNetwork: true
hostPID: true
initContainers:
- name: install-cni
- image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }}
+ image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
command: ["/kube-ovn/install-cni.sh"]
+ securityContext:
+ runAsUser: 0
+ privileged: true
volumeMounts:
- mountPath: /etc/cni/net.d
name: cni-conf
@@ -120,16 +119,18 @@ spec:
name: cni-bin
containers:
- name: cni-server
- image: {{ kube_ovn_cni_image_repo }}:{{ kube_ovn_cni_image_tag }}
+ image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
command:
- sh
- /kube-ovn/start-cniserver.sh
args:
- --enable-mirror={{ traffic_mirror }}
+ - --encap-checksum={{ encap_checksum }}
+ - --service-cluster-ip-range={{ kube_service_addresses }}
securityContext:
- runAsUser: 0
- privileged: true
+ capabilities:
+ add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"]
env:
- name: POD_IP
valueFrom:
@@ -142,6 +143,11 @@ spec:
volumeMounts:
- mountPath: /run/openvswitch
name: host-run-ovs
+ - mountPath: /run/ovn
+ name: host-run-ovn
+ - mountPath: /var/run/netns
+ name: host-ns
+ mountPropagation: HostToContainer
readinessProbe:
exec:
command:
@@ -163,14 +169,165 @@ spec:
periodSeconds: 7
failureThreshold: 5
nodeSelector:
- beta.kubernetes.io/os: "linux"
+ kubernetes.io/os: "linux"
volumes:
- name: host-run-ovs
hostPath:
path: /run/openvswitch
+ - name: host-run-ovn
+ hostPath:
+ path: /run/ovn
- name: cni-conf
hostPath:
path: /etc/cni/net.d
- name: cni-bin
hostPath:
- path: /opt/cni/bin
\ No newline at end of file
+ path: /opt/cni/bin
+ - name: host-ns
+ hostPath:
+ path: /var/run/netns
+
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+ name: kube-ovn-pinger
+ namespace: kube-ovn
+ annotations:
+ kubernetes.io/description: |
+ This daemon set launches the openvswitch daemon.
+spec:
+ selector:
+ matchLabels:
+ app: kube-ovn-pinger
+ updateStrategy:
+ type: RollingUpdate
+ template:
+ metadata:
+ labels:
+ app: kube-ovn-pinger
+ component: network
+ type: infra
+ spec:
+ tolerations:
+ - operator: Exists
+ effect: NoSchedule
+ serviceAccountName: ovn
+ hostPID: true
+ containers:
+ - name: pinger
+ image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
+ command: ["/kube-ovn/kube-ovn-pinger", "--external-address=114.114.114.114"]
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ securityContext:
+ runAsUser: 0
+ privileged: false
+ env:
+ - name: POD_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.podIP
+ - name: HOST_IP
+ valueFrom:
+ fieldRef:
+ fieldPath: status.hostIP
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+ volumeMounts:
+ - mountPath: /lib/modules
+ name: host-modules
+ readOnly: true
+ - mountPath: /run/openvswitch
+ name: host-run-ovs
+ - mountPath: /var/run/openvswitch
+ name: host-run-ovs
+ - mountPath: /var/run/ovn
+ name: host-run-ovn
+ - mountPath: /sys
+ name: host-sys
+ readOnly: true
+ - mountPath: /etc/openvswitch
+ name: host-config-openvswitch
+ - mountPath: /var/log/openvswitch
+ name: host-log-ovs
+ - mountPath: /var/log/ovn
+ name: host-log-ovn
+ resources:
+ requests:
+ cpu: {{ kube_ovn_pinger_cpu_request }}
+ memory: {{ kube_ovn_pinger_memory_request }}
+ limits:
+ cpu: {{ kube_ovn_pinger_cpu_limit }}
+ memory: {{ kube_ovn_pinger_memory_limit }}
+ nodeSelector:
+ kubernetes.io/os: "linux"
+ volumes:
+ - name: host-modules
+ hostPath:
+ path: /lib/modules
+ - name: host-run-ovs
+ hostPath:
+ path: /run/openvswitch
+ - name: host-run-ovn
+ hostPath:
+ path: /run/ovn
+ - name: host-sys
+ hostPath:
+ path: /sys
+ - name: host-config-openvswitch
+ hostPath:
+ path: /etc/origin/openvswitch
+ - name: host-log-ovs
+ hostPath:
+ path: /var/log/openvswitch
+ - name: host-log-ovn
+ hostPath:
+ path: /var/log/ovn
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: kube-ovn-pinger
+ namespace: kube-ovn
+ labels:
+ app: kube-ovn-pinger
+spec:
+ selector:
+ app: kube-ovn-pinger
+ ports:
+ - port: 8080
+ name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: kube-ovn-controller
+ namespace: kube-ovn
+ labels:
+ app: kube-ovn-controller
+spec:
+ selector:
+ app: kube-ovn-controller
+ ports:
+ - port: 10660
+ name: metrics
+---
+kind: Service
+apiVersion: v1
+metadata:
+ name: kube-ovn-cni
+ namespace: kube-ovn
+ labels:
+ app: kube-ovn-cni
+spec:
+ selector:
+ app: kube-ovn-cni
+ ports:
+ - port: 10665
+ name: metrics
\ No newline at end of file
diff --git a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2 b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
index f86967b35..e4ee7e12c 100644
--- a/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
+++ b/roles/network_plugin/kube-ovn/templates/cni-ovn.yml.j2
@@ -29,6 +29,7 @@ rules:
- "kubeovn.io"
resources:
- subnets
+ - subnets/status
- ips
verbs:
- "*"
@@ -55,6 +56,7 @@ rules:
- services
- endpoints
- statefulsets
+ - daemonsets
verbs:
- get
- list
@@ -97,6 +99,7 @@ spec:
type: ClusterIP
selector:
app: ovn-central
+ ovn-nb-leader: "true"
sessionAffinity: None
---
@@ -114,6 +117,7 @@ spec:
type: ClusterIP
selector:
app: ovn-central
+ ovn-sb-leader: "true"
sessionAffinity: None
---
@@ -152,17 +156,30 @@ spec:
matchLabels:
app: ovn-central
topologyKey: kubernetes.io/hostname
+ priorityClassName: system-cluster-critical
serviceAccountName: ovn
hostNetwork: true
containers:
- name: ovn-central
- image: {{ kube_ovn_db_image_repo }}:{{ kube_ovn_db_image_tag }}
+ image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
+ command: ["/kube-ovn/start-db.sh"]
+ securityContext:
+ capabilities:
+ add: ["SYS_NICE"]
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
resources:
requests:
cpu: {{ kube_ovn_db_cpu_request }}
@@ -171,47 +188,55 @@ spec:
cpu: {{ kube_ovn_db_cpu_limit }}
memory: {{ kube_ovn_db_memory_limit }}
volumeMounts:
- - mountPath: /run/openvswitch
- name: host-run-ovs
- mountPath: /var/run/openvswitch
name: host-run-ovs
+ - mountPath: /var/run/ovn
+ name: host-run-ovn
- mountPath: /sys
name: host-sys
readOnly: true
- mountPath: /etc/openvswitch
name: host-config-openvswitch
- mountPath: /var/log/openvswitch
- name: host-log
+ name: host-log-ovs
+ - mountPath: /var/log/ovn
+ name: host-log-ovn
readinessProbe:
exec:
command:
- sh
- - /root/ovn-is-leader.sh
+ - /kube-ovn/ovn-is-leader.sh
periodSeconds: 3
livenessProbe:
exec:
command:
- sh
- - /root/ovn-healthcheck.sh
+ - /kube-ovn/ovn-healthcheck.sh
initialDelaySeconds: 30
periodSeconds: 7
failureThreshold: 5
nodeSelector:
- beta.kubernetes.io/os: "linux"
+ kubernetes.io/os: "linux"
kube-ovn/role: "master"
volumes:
- name: host-run-ovs
hostPath:
path: /run/openvswitch
+ - name: host-run-ovn
+ hostPath:
+ path: /run/ovn
- name: host-sys
hostPath:
path: /sys
- name: host-config-openvswitch
hostPath:
path: /etc/origin/openvswitch
- - name: host-log
+ - name: host-log-ovs
hostPath:
path: /var/log/openvswitch
+ - name: host-log-ovn
+ hostPath:
+ path: /var/log/ovn
---
kind: DaemonSet
@@ -227,7 +252,7 @@ spec:
matchLabels:
app: ovs
updateStrategy:
- type: RollingUpdate
+ type: OnDelete
template:
metadata:
labels:
@@ -238,13 +263,15 @@ spec:
tolerations:
- operator: Exists
effect: NoSchedule
+ priorityClassName: system-cluster-critical
serviceAccountName: ovn
hostNetwork: true
hostPID: true
containers:
- name: openvswitch
- image: {{ kube_ovn_node_image_repo }}:{{ kube_ovn_node_image_tag }}
+ image: {{ kube_ovn_container_image_repo }}:{{ kube_ovn_container_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }}
+ command: ["/kube-ovn/start-ovs.sh"]
securityContext:
runAsUser: 0
privileged: true
@@ -257,28 +284,30 @@ spec:
- mountPath: /lib/modules
name: host-modules
readOnly: true
- - mountPath: /run/openvswitch
- name: host-run-ovs
- mountPath: /var/run/openvswitch
name: host-run-ovs
+ - mountPath: /var/run/ovn
+ name: host-run-ovn
- mountPath: /sys
name: host-sys
readOnly: true
- mountPath: /etc/openvswitch
name: host-config-openvswitch
- mountPath: /var/log/openvswitch
- name: host-log
+ name: host-log-ovs
+ - mountPath: /var/log/ovn
+ name: host-log-ovn
readinessProbe:
exec:
command:
- sh
- - /root/ovs-healthcheck.sh
+ - /kube-ovn/ovs-healthcheck.sh
periodSeconds: 5
livenessProbe:
exec:
command:
- sh
- - /root/ovs-healthcheck.sh
+ - /kube-ovn/ovs-healthcheck.sh
initialDelaySeconds: 10
periodSeconds: 5
failureThreshold: 5
@@ -290,7 +319,7 @@ spec:
cpu: {{ kube_ovn_node_cpu_limit }}
memory: {{ kube_ovn_node_memory_limit }}
nodeSelector:
- beta.kubernetes.io/os: "linux"
+ kubernetes.io/os: "linux"
volumes:
- name: host-modules
hostPath:
@@ -298,12 +327,18 @@ spec:
- name: host-run-ovs
hostPath:
path: /run/openvswitch
+ - name: host-run-ovn
+ hostPath:
+ path: /run/ovn
- name: host-sys
hostPath:
path: /sys
- name: host-config-openvswitch
hostPath:
path: /etc/origin/openvswitch
- - name: host-log
+ - name: host-log-ovs
+ hostPath:
+ path: /var/log/openvswitch
+ - name: host-log-ovn
hostPath:
- path: /var/log/openvswitch
\ No newline at end of file
+ path: /var/log/ovn
\ No newline at end of file
diff --git a/tests/files/packet_centos7-kube-ovn.yml b/tests/files/packet_centos8-kube-ovn.yml
similarity index 55%
rename from tests/files/packet_centos7-kube-ovn.yml
rename to tests/files/packet_centos8-kube-ovn.yml
index e954a887b..6fab50204 100644
--- a/tests/files/packet_centos7-kube-ovn.yml
+++ b/tests/files/packet_centos8-kube-ovn.yml
@@ -1,12 +1,9 @@
---
# Instance settings
-cloud_image: centos-7
+cloud_image: centos-8
mode: default
# Kubespray settings
kube_network_plugin: kube-ovn
deploy_netchecker: true
-dns_min_replicas: 1
-
-# Temp set k8s ver to 1.16.8
-kube_version: v1.16.7
\ No newline at end of file
+dns_min_replicas: 1
\ No newline at end of file
--
GitLab