diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fc4582394f4b90e5f9d0a2af88d9d4c6dd60e16..de4285c90576f26b97da8418ea4a5e54787778ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -300,6 +300,10 @@ before_script: # stage: deploy-special MOVED_TO_GROUP_VARS: "true" +.centos7_multus_calico_variables: ¢os7_multus_calico_variables +# stage: deploy-part2 + MOVED_TO_GROUP_VARS: "true" + .coreos_alpha_weave_ha_variables: &coreos_alpha_weave_ha_variables # stage: deploy-special MOVED_TO_GROUP_VARS: "true" @@ -638,6 +642,17 @@ gce_centos7-kube-router: except: ['triggers'] only: ['master', /^pr-.*$/] +gce_centos7-multus-calico: + stage: deploy-part2 + <<: *job + <<: *gce + variables: + <<: *gce_variables + <<: *centos7_multus_calico_variables + when: manual + except: ['triggers'] + only: ['master', /^pr-.*$/] + gce_opensuse-canal: stage: deploy-part2 <<: *job diff --git a/README.md b/README.md index 2c7175936017c67d808bfde459faf97fdcd0c69b..f9d197184bf2a2c5277e308e57a68e69f73525cb 100644 --- a/README.md +++ b/README.md @@ -125,6 +125,7 @@ Supported Components - [weave](https://github.com/weaveworks/weave) v2.4.1 - [kube-router](https://github.com/cloudnativelabs/kube-router) v0.2.1 - Application + - [multus](https://github.com/intel/multus-cni) v3.1 - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11 - [cert-manager](https://github.com/jetstack/cert-manager) v0.5.0 - [coredns](https://github.com/coredns/coredns) v1.2.5 @@ -176,6 +177,8 @@ You can choose between 6 network plugins. (default: `calico`, except Vagrant use iptables for network policies, and BGP for ods L3 networking (with optionally BGP peering with out-of-cluster BGP peers). It can also optionally advertise routes to Kubernetes cluster Pods CIDRs, ClusterIPs, ExternalIPs and LoadBalancerIPs. +- [multus](docs/multus.md): Multus is a meta CNI plugin that provides multiple network interface support to pods. For each interface Multus delegates CNI calls to secondary CNI plugins such as Calico, macvlan, etc. + The choice is defined with the variable `kube_network_plugin`. There is also an option to leverage built-in cloud provider networking instead. See also [Network checker](docs/netcheck.md). diff --git a/Vagrantfile b/Vagrantfile index 1c0d6d7b91952e4a311f47f9b81967fce4e69d58..de612516ffd191c19e6b33fb1a33f7cf60622f81 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -35,6 +35,8 @@ $forwarded_ports = {} $subnet = "172.17.8" $os = "ubuntu1804" $network_plugin = "flannel" +# Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni +$multi_networking = false # The first three nodes are etcd servers $etcd_instances = $num_instances # The first two nodes are kube masters @@ -140,7 +142,8 @@ Vagrant.configure("2") do |config| "ip": ip, "local_release_dir" => $local_release_dir, "download_run_once": "False", - "kube_network_plugin": $network_plugin + "kube_network_plugin": $network_plugin, + "kube_network_plugin_multus": $multi_networking } config.vm.network :private_network, ip: ip diff --git a/docs/multus.md b/docs/multus.md new file mode 100644 index 0000000000000000000000000000000000000000..2d46135f553f106adb2a35583efa7526c84da7a0 --- /dev/null +++ b/docs/multus.md @@ -0,0 +1,73 @@ +Multus +=========== + +Multus is a meta CNI plugin that provides multiple network interface support to +pods. For each interface, Multus delegates CNI calls to secondary CNI plugins +such as Calico, macvlan, etc. + +See [multus documentation](https://github.com/intel/multus-cni). + +## Multus installation + +Since Multus itself does not implement networking, it requires a master plugin, which is specified through the variable `kube_network_plugin`. To enable Multus an additional variable `kube_network_plugin_multus` must be set to `true`. For example, +``` +kube_network_plugin: calico +kube_network_plugin_multus: true +``` +will install Multus and Calico and configure Multus to use Calico as the primary network plugin. + +## Using Multus + +Once Multus is installed, you can create CNI configurations (as a CRD objects) for additional networks, in this case a macvlan CNI configuration is defined. You may replace the config field with any valid CNI configuration where the CNI binary is available on the nodes. + +``` +cat <<EOF | kubectl create -f - +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: macvlan-conf +spec: + config: '{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "192.168.1.0/24", + "rangeStart": "192.168.1.200", + "rangeEnd": "192.168.1.216", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.1.1" + } + }' +EOF +``` + +You may then create a pod with and additional interface that connects to this network using annotations. The annotation correlates to the name in the NetworkAttachmentDefinition above. + +``` +cat <<EOF | kubectl create -f - +apiVersion: v1 +kind: Pod +metadata: + name: samplepod + annotations: + k8s.v1.cni.cncf.io/networks: macvlan-conf +spec: + containers: + - name: samplepod + command: ["/bin/bash", "-c", "sleep 2000000000000"] + image: dougbtv/centos-network +EOF +``` + +You may now inspect the pod and see that there is an additional interface configured: + +``` +$ kubectl exec -it samplepod -- ip a +``` + +For more details on how to use Multus, please visit https://github.com/intel/multus-cni diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index b375e07ffd1c5ffcf32d144f74bcac24671fe3a3..aca85066882d343c27a6169a0c58480ab5ccf5b1 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -59,6 +59,7 @@ pod_infra_version: 3.1 contiv_version: 1.2.1 cilium_version: "v1.3.0" kube_router_version: "v0.2.1" +multus_version: "v3.1.autoconf" # Download URLs kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm" @@ -160,6 +161,8 @@ cilium_image_repo: "docker.io/cilium/cilium" cilium_image_tag: "{{ cilium_version }}" kube_router_image_repo: "cloudnativelabs/kube-router" kube_router_image_tag: "{{ kube_router_version }}" +multus_image_repo: "docker.io/nfvpe/multus" +multus_image_tag: "{{ multus_version }}" nginx_image_repo: nginx nginx_image_tag: 1.13 dnsmasq_version: 2.78 @@ -290,6 +293,15 @@ downloads: groups: - k8s-cluster + multus: + enabled: "{{ kube_network_plugin_multus }}" + container: true + repo: "{{ multus_image_repo }}" + tag: "{{ multus_image_tag }}" + sha256: "{{ multus_digest_checksum|default(None) }}" + groups: + - k8s-cluster + flannel: enabled: "{{ kube_network_plugin == 'flannel' or kube_network_plugin == 'canal' }}" container: true diff --git a/roles/kubernetes-apps/network_plugin/meta/main.yml b/roles/kubernetes-apps/network_plugin/meta/main.yml index c88dbf0153a1a2d35f045d6e31b9bd8686c6c4e4..8d2a5be1b989a7e2cea5d3b285c3c37db0fa31f7 100644 --- a/roles/kubernetes-apps/network_plugin/meta/main.yml +++ b/roles/kubernetes-apps/network_plugin/meta/main.yml @@ -34,3 +34,8 @@ dependencies: when: kube_network_plugin == 'kube-router' tags: - kube-router + + - role: kubernetes-apps/network_plugin/multus + when: kube_network_plugin_multus + tags: + - multus diff --git a/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml b/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..9d7669cc7edaf722a05a738b7dbaf72b0a306862 --- /dev/null +++ b/roles/kubernetes-apps/network_plugin/multus/tasks/main.yml @@ -0,0 +1,11 @@ +--- +- name: Multus | Start resources + kube: + name: "{{item.item.name}}" + namespace: "kube-system" + kubectl: "{{bin_dir}}/kubectl" + resource: "{{item.item.type}}" + filename: "{{kube_config_dir}}/{{item.item.file}}" + state: "latest" + with_items: "{{ multus_manifest_1.results }} + {{multus_manifest_2.results }}" + when: inventory_hostname == groups['kube-master'][0] and not item|skipped diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 48bf477cb4dcf914929ff41a21577f094dee91d0..1f2d7aa380ff174d45e576743c7679ac72f0d203 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -103,6 +103,7 @@ kube_users: # Choose network plugin (cilium, calico, weave or flannel) # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico +kube_network_plugin_multus: false # Determines if calico-rr group exists peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" diff --git a/roles/network_plugin/contiv/defaults/main.yml b/roles/network_plugin/contiv/defaults/main.yml index 5a3778937263c249d0639666684650433c8cf012..82316357ceeed65868e14d595d605a15c5f6359d 100644 --- a/roles/network_plugin/contiv/defaults/main.yml +++ b/roles/network_plugin/contiv/defaults/main.yml @@ -4,7 +4,7 @@ contiv_config_dir: "{{ kube_config_dir }}/contiv" contiv_etcd_conf_dir: "/etc/contiv/etcd" contiv_etcd_data_dir: "/var/lib/etcd/contiv-data" contiv_netmaster_port: 9999 -contiv_cni_version: 0.1.0 +contiv_cni_version: 0.3.1 # No need to download it by default, but must be defined contiv_etcd_image_repo: "{{ etcd_image_repo }}" diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml index a0fae72076ad1a97e7c648875e5c477b3b4904f9..12064237101c1fb9437e56c7150e5d836e5f085a 100644 --- a/roles/network_plugin/meta/main.yml +++ b/roles/network_plugin/meta/main.yml @@ -37,3 +37,8 @@ dependencies: - role: network_plugin/cloud when: kube_network_plugin == 'cloud' + + - role: network_plugin/multus + when: kube_network_plugin_multus + tags: + - multus diff --git a/roles/network_plugin/multus/defaults/main.yml b/roles/network_plugin/multus/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..2fb72310349a609e1b85b0b260e23a4210cc0d02 --- /dev/null +++ b/roles/network_plugin/multus/defaults/main.yml @@ -0,0 +1,7 @@ +--- +multus_conf_file: "auto" +multus_cni_conf_dir_host: "/etc/cni/net.d" +multus_cni_bin_dir_host: "/opt/cni/bin" +multus_cni_conf_dir: "{{ ('/host', multus_cni_conf_dir_host) | join }}" +multus_cni_bin_dir: "{{ ('/host', multus_cni_bin_dir_host) | join }}" +multus_kubeconfig_file_host: "{{ (multus_cni_conf_dir_host, '/multus.d/multus.kubeconfig') | join }}" diff --git a/roles/network_plugin/multus/files/multus-clusterrole.yml b/roles/network_plugin/multus/files/multus-clusterrole.yml new file mode 100644 index 0000000000000000000000000000000000000000..337775be2428295ff5c5b94d9b3f72e31ea45448 --- /dev/null +++ b/roles/network_plugin/multus/files/multus-clusterrole.yml @@ -0,0 +1,16 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: multus +rules: +- apiGroups: + - '*' + resources: + - '*' + verbs: + - '*' +- nonResourceURLs: + - '*' + verbs: + - '*' diff --git a/roles/network_plugin/multus/files/multus-clusterrolebinding.yml b/roles/network_plugin/multus/files/multus-clusterrolebinding.yml new file mode 100644 index 0000000000000000000000000000000000000000..5980330ebd385529e2c55a0aa2f4a0ddb45d4ca6 --- /dev/null +++ b/roles/network_plugin/multus/files/multus-clusterrolebinding.yml @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: multus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multus +subjects: +- kind: ServiceAccount + name: multus + namespace: kube-system diff --git a/roles/network_plugin/multus/files/multus-crd.yml b/roles/network_plugin/multus/files/multus-crd.yml new file mode 100644 index 0000000000000000000000000000000000000000..eab4406e21f61536d7f42a2b7d39498cd624f56c --- /dev/null +++ b/roles/network_plugin/multus/files/multus-crd.yml @@ -0,0 +1,22 @@ +--- +kind: CustomResourceDefinition +apiVersion: apiextensions.k8s.io/v1beta1 +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + version: v1 + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - net-attach-def + validation: + openAPIV3Schema: + properties: + spec: + properties: + config: + type: string diff --git a/roles/network_plugin/multus/files/multus-serviceaccount.yml b/roles/network_plugin/multus/files/multus-serviceaccount.yml new file mode 100644 index 0000000000000000000000000000000000000000..62423082ca06e928d0e3d09274e3f6b04487b633 --- /dev/null +++ b/roles/network_plugin/multus/files/multus-serviceaccount.yml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multus + namespace: kube-system diff --git a/roles/network_plugin/multus/tasks/main.yml b/roles/network_plugin/multus/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..7f603973dbc3aa874f24e75f3fa6dd5577d60638 --- /dev/null +++ b/roles/network_plugin/multus/tasks/main.yml @@ -0,0 +1,19 @@ +--- +- name: Multus | Copy manifest files + copy: + src: "{{ item.file }}" + dest: "{{ kube_config_dir }}" + with_items: + - {name: multus-crd, file: multus-crd.yml, type: customresourcedefinition} + - {name: multus-serviceaccount, file: multus-serviceaccount.yml, type: serviceaccount} + - {name: multus-clusterrole, file: multus-clusterrole.yml, type: clusterrole} + - {name: multus-clusterrolebinding, file: multus-clusterrolebinding.yml, type: clusterrolebinding} + register: multus_manifest_1 + +- name: Multus | Copy manifest templates + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + with_items: + - {name: multus-daemonset, file: multus-daemonset.yml, type: daemonset} + register: multus_manifest_2 diff --git a/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 b/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..11cf427d047db19c174dfea345047adbe132bddc --- /dev/null +++ b/roles/network_plugin/multus/templates/multus-daemonset.yml.j2 @@ -0,0 +1,54 @@ +--- +kind: DaemonSet +apiVersion: extensions/v1beta1 +metadata: + name: kube-multus-ds-amd64 + namespace: kube-system + labels: + tier: node + app: multus +spec: + template: + metadata: + labels: + tier: node + app: multus + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: multus + containers: + - name: kube-multus + image: {{ multus_image_repo }}:{{ multus_image_tag }} + command: ["/entrypoint.sh"] + args: + - "--cni-conf-dir={{ multus_cni_conf_dir }}" + - "--cni-bin-dir={{ multus_cni_bin_dir }}" + - "--multus-conf-file={{ multus_conf_file }}" + - "--multus-kubeconfig-file-host={{ multus_kubeconfig_file_host }}" + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + volumeMounts: + - name: cni + mountPath: {{ multus_cni_conf_dir }} + - name: cnibin + mountPath: {{ multus_cni_bin_dir }} + volumes: + - name: cni + hostPath: + path: {{ multus_cni_conf_dir_host }} + - name: cnibin + hostPath: + path: {{ multus_cni_bin_dir_host }} diff --git a/tests/files/gce_centos7-multus-calico.yml b/tests/files/gce_centos7-multus-calico.yml new file mode 100644 index 0000000000000000000000000000000000000000..57615e285ea5db7f94bd5db63db6c419c55a0878 --- /dev/null +++ b/tests/files/gce_centos7-multus-calico.yml @@ -0,0 +1,12 @@ +# Instance settings +cloud_image_family: centos-7 +cloud_region: us-central1-c +cloud_machine_type: "n1-standard-1" +mode: default + +# Deployment settings +kube_network_plugin_multus: true +kube_network_plugin: calico +deploy_netchecker: true +kubedns_min_replicas: 1 +cloud_provider: gce diff --git a/tests/testcases/040_check-network-adv.yml b/tests/testcases/040_check-network-adv.yml index 709e8d8b4d317646362d0b1e1c609a1d8c7ecb87..819a7a4851475a45cb1a5471378bb8bbae95ba0a 100644 --- a/tests/testcases/040_check-network-adv.yml +++ b/tests/testcases/040_check-network-adv.yml @@ -80,3 +80,68 @@ run_once: true when: - agents.content == '{}' + + - name: Create macvlan network conf + # We cannot use only shell: below because Ansible will render the text + # with leading spaces, which means the shell will never find the string + # EOF at the beginning of a line. We can avoid Ansible's unhelpful + # heuristics by using the cmd parameter like this: + shell: + cmd: | + cat <<EOF | {{ bin_dir }}/kubectl create -f - + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + name: macvlan-conf + spec: + config: '{ + "cniVersion": "0.3.0", + "type": "macvlan", + "master": "eth0", + "mode": "bridge", + "ipam": { + "type": "host-local", + "subnet": "192.168.1.0/24", + "rangeStart": "192.168.1.200", + "rangeEnd": "192.168.1.216", + "routes": [ + { "dst": "0.0.0.0/0" } + ], + "gateway": "192.168.1.1" + } + }' + EOF + when: + - kube_network_plugin_multus|default(false) + + - name: Annotate pod with macvlan network + # We cannot use only shell: below because Ansible will render the text + # with leading spaces, which means the shell will never find the string + # EOF at the beginning of a line. We can avoid Ansible's unhelpful + # heuristics by using the cmd parameter like this: + shell: + cmd: | + cat <<EOF | {{ bin_dir }}/kubectl create -f - + apiVersion: v1 + kind: Pod + metadata: + name: samplepod + annotations: + k8s.v1.cni.cncf.io/networks: macvlan-conf + spec: + containers: + - name: samplepod + command: ["/bin/bash", "-c", "sleep 2000000000000"] + image: dougbtv/centos-network + EOF + when: + - kube_network_plugin_multus|default(false) + + - name: Check secondary macvlan interface + shell: "{{ bin_dir }}/kubectl exec samplepod -- ip addr show dev net1" + register: output + until: output.rc == 0 + retries: 90 + changed_when: false + when: + - kube_network_plugin_multus|default(false)