diff --git a/docs/calico.md b/docs/calico.md index f090ca984f7f3458d783ab5267a7d7635e7fbc94..13c1856c75bcdfd405c07d265ad8f26b94f92033 100644 --- a/docs/calico.md +++ b/docs/calico.md @@ -260,3 +260,73 @@ calico_ipam_host_local: true ``` Refer to Project Calico section [Using host-local IPAM](https://docs.projectcalico.org/reference/cni-plugin/configuration#using-host-local-ipam) for further information. + +## eBPF Support + +Calico supports eBPF for its data plane see [an introduction to the Calico eBPF Dataplane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/) for further information. + +Note that it is advisable to always use the latest version of Calico when using the eBPF dataplane. + +### Enabling eBPF support + +To enable the eBPF dataplane support ensure you add the following to your inventory. Note that the `kube-proxy` is incompatible with running Calico in eBPF mode and the kube-proxy should be removed from the system. + +```yaml +calico_bpf_enabled: true +kube_proxy_remove: true +``` + +### Cleaning up after kube-proxy + +Calico node cannot clean up after kube-proxy has run in ipvs mode. If you are converting an existing cluster to eBPF you will need to ensure the `kube-proxy` DaemonSet is deleted and that ipvs rules are cleaned. + +To check that kube-proxy was running in ipvs mode: + +```ShellSession +# ipvsadm -l +``` + +To clean up any ipvs leftovers: + +```ShellSession +# ipvsadm -C +``` + +### Calico access to the kube-api + +Calico node, typha and kube-controllers need to be able to talk to the kubernetes API. Please reference the [Enabling eBPF Calico Docs](https://docs.projectcalico.org/maintenance/ebpf/enabling-bpf) for guidelines on how to do this. + +Kubespray sets up the `kubernetes-services-endpoint` configmap based on the contents of the `loadbalancer_apiserver` inventory variable documented in [HA Mode](./ha-mode.md). + +If no external loadbalancer is used, Calico eBPF can also use the localhost loadbalancer option. In this case Calico Automatic Host Endpoints need to be enabled to allow services like `coredns` and `metrics-server` to communicate with the kubernetes host endpoint. See [this blog post](https://www.projectcalico.org/securing-kubernetes-nodes-with-calico-automatic-host-endpoints/) on enabling automatic host endpoints. + +```yaml +loadbalancer_apiserver_localhost: true +use_localhost_as_kubeapi_loadbalancer: true +``` + +### Tunneled versus Direct Server Return + +By default Calico usese Tunneled service mode but it can use direct server return (DSR) in order to optimize the return path for a service. + +To configure DSR: + +```yaml +calico_bpf_service_mode: "DSR" +``` + +### eBPF Logging and Troubleshooting + +In order to enable Calico eBPF mode logging: + +```yaml +calico_bpf_log_level: "Debug" +``` + +To view the logs you need to use the `tc` command to read the kernel trace buffer: + +```ShellSession +tc exec bpf debug +``` + +Please see [Calico eBPF troubleshooting guide](https://docs.projectcalico.org/maintenance/troubleshoot/troubleshoot-ebpf#ebpf-program-debug-logs). diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 00851618153b3fd89c0531116f107caf6de9314c..1074b09b649165495790ae3753a294ac59d55693 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -69,6 +69,7 @@ quay_image_repo: "quay.io" calico_version: "v3.18.4" calico_ctl_version: "{{ calico_version }}" calico_cni_version: "{{ calico_version }}" +calico_flexvol_version: "{{ calico_version }}" calico_policy_version: "{{ calico_version }}" calico_typha_version: "{{ calico_version }}" typha_enabled: false @@ -439,6 +440,8 @@ calico_node_image_repo: "{{ quay_image_repo }}/calico/node" calico_node_image_tag: "{{ calico_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}" calico_cni_image_repo: "{{ quay_image_repo }}/calico/cni" calico_cni_image_tag: "{{ calico_cni_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}" +calico_flexvol_image_repo: "{{ quay_image_repo }}/calico/pod2daemon-flexvol" +calico_flexvol_image_tag: "{{ calico_flexvol_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}" calico_policy_image_repo: "{{ quay_image_repo }}/calico/kube-controllers" calico_policy_image_tag: "{{ calico_policy_version }}{%- if image_arch != 'amd64' -%}-{{ image_arch }}{%- endif -%}" calico_typha_image_repo: "{{ quay_image_repo }}/calico/typha" diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index 186e9db0b452fe548992942d65f2aa3ec723c0e5..ae618243de1313ce399485689f9ac4d484fe1165 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -47,6 +47,25 @@ - kube_network_plugin == 'calico' - not ignore_assert_errors +- name: Stop if supported Calico versions + assert: + that: + - "calico_version in calico_crds_archive_checksums.keys()" + msg: "Calico version not supported {{ calico_version }} not in {{ calico_crds_archive_checksums.keys() }}" + when: + - kube_network_plugin == 'calico' + - not ignore_assert_errors + +- name: Stop if kube-proxy is enabled when using eBPF dataplane + assert: + that: + - kube_proxy_remove + msg: "kube-proxy needs to be disabled when using Calico with eBPF dataplane" + when: + - calico_bpf_enabled | default(false) + - kube_network_plugin == 'calico' + - not ignore_assert_errors + - name: Stop if unsupported version of Kubernetes assert: that: kube_version is version(kube_version_min_required, '>=') diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index ce6d12292bd02740c278b592302037c430fdd56a..c95833f1b93507b314a6cc7a1e233311790a6e9d 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -32,6 +32,12 @@ calico_advertise_service_external_ips: [] # Adveritse Service LoadBalancer IPs calico_advertise_service_loadbalancer_ips: [] +# Calico eBPF support +calico_bpf_enabled: false +calico_bpf_log_level: "" +# Valid option for service mode: Tunnel (default), DSR=Direct Server Return +calico_bpf_service_mode: Tunnel + # Limits for apps calico_node_memory_limit: 500M calico_node_cpu_limit: 300m @@ -91,6 +97,8 @@ kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem # Use typha (only with kdd) typha_enabled: false +typha_prometheusmetricsenabled: false +typha_prometheusmetricsport: 9093 # Scaling typha: 1 replica per 100 nodes is adequate # Number of typha replicas @@ -106,3 +114,7 @@ calico_feature_control: {} # Calico default BGP port calico_bgp_listen_port: 179 + +# Calico FelixConfiguration options +calico_felix_reporting_interval: 0s +calico_felix_log_severity_screen: Info diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml index 3e081398af646a41288e27c0229a02e022a84f66..4c196acc5abb6b74109720a2c89f650426e8c936 100644 --- a/roles/network_plugin/calico/tasks/install.yml +++ b/roles/network_plugin/calico/tasks/install.yml @@ -131,6 +131,27 @@ - inventory_hostname in groups['kube_control_plane'] - calico_datastore == "kdd" +- name: Calico | Configure calico FelixConfiguration + command: + cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" + stdin: "{{ stdin is string | ternary(stdin, stdin|to_json) }}" + vars: + stdin: > + { "kind": "FelixConfiguration", + "apiVersion": "projectcalico.org/v3", + "metadata": { + "name": "default", + }, + "spec": { + "ipipEnabled": {{ calico_ipip_mode != 'Never' | bool }}, + "reportingInterval": "{{ calico_felix_reporting_interval }}", + "bpfLogLevel": "{{ calico_bpf_log_level }}", + "bpfEnabled": {{ calico_bpf_enabled | bool }}, + "bpfExternalServiceMode": "{{ calico_bpf_service_mode }}", + "logSeverityScreen": "{{ calico_felix_log_severity_screen }}" }} + when: + - inventory_hostname == groups['kube_control_plane'][0] + - name: Calico | Configure calico network pool command: cmd: "{{ bin_dir }}/calicoctl.sh apply -f -" @@ -302,6 +323,7 @@ - {name: calico, file: calico-node-sa.yml, type: sa} - {name: calico, file: calico-cr.yml, type: clusterrole} - {name: calico, file: calico-crb.yml, type: clusterrolebinding} + - {name: kubernetes-services-endpoint, file: kubernetes-services-endpoint.yml, type: cm } register: calico_node_manifests when: - inventory_hostname in groups['kube_control_plane'] diff --git a/roles/network_plugin/calico/templates/calico-cr.yml.j2 b/roles/network_plugin/calico/templates/calico-cr.yml.j2 index b66b43d27c7aeb210fb459932fb56d8cf62b6b6d..e5238d8d30020242359865386bf09a7d5b57ee72 100644 --- a/roles/network_plugin/calico/templates/calico-cr.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2 @@ -28,6 +28,7 @@ rules: resources: - nodes/status verbs: + # Needed for clearing NodeNetworkUnavailable flag. - patch {% if calico_datastore == "etcd" %} - apiGroups: diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 155189b9a9f6253b5750d35414f067e5a23d3deb..651711e78e6447595bd213ecfbf9b22f2d7c6e8c 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -44,6 +44,11 @@ spec: - name: upgrade-ipam image: {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: - name: KUBERNETES_NODE_NAME valueFrom: @@ -94,12 +99,26 @@ spec: name: cni-bin-dir securityContext: privileged: true + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: {{ calico_flexvol_image_repo }}:{{ calico_flexvol_image_tag }} + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + securityContext: + privileged: true containers: # Runs calico/node container on each Kubernetes node. This # container programs network policy and routes on each # host. - name: calico-node image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }} + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # The location of the Calico etcd cluster. {% if calico_datastore == "etcd" %} @@ -231,8 +250,6 @@ spec: {% if calico_ip_auto_method is defined %} - name: IP_AUTODETECTION_METHOD value: "{{ calico_ip_auto_method }}" - - name: IP - value: "autodetect" {% else %} - name: NODEIP valueFrom: @@ -240,9 +257,9 @@ spec: fieldPath: status.hostIP - name: IP_AUTODETECTION_METHOD value: "can-reach=$(NODEIP)" +{% endif %} - name: IP value: "autodetect" -{% endif %} {% if enable_dual_stack_networks %} - name: IP6 value: autodetect @@ -286,10 +303,10 @@ spec: {% if calico_network_backend|default("bird") == "bird" %} - -bird-live {% endif %} - initialDelaySeconds: 5 + periodSeconds: 10 + initialDelaySeconds: 10 failureThreshold: 6 readinessProbe: - failureThreshold: 6 exec: command: - /bin/calico-node @@ -297,18 +314,22 @@ spec: - -bird-ready {% endif %} - -felix-ready + periodSeconds: 10 + failureThreshold: 6 volumeMounts: - mountPath: /lib/modules name: lib-modules readOnly: true - mountPath: /var/run/calico name: var-run-calico + readOnly: false - mountPath: /var/lib/calico name: var-lib-calico readOnly: false {% if calico_datastore == "etcd" %} - mountPath: /calico-secrets name: etcd-certs + readOnly: true {% endif %} - name: xtables-lock mountPath: /run/xtables.lock @@ -324,7 +345,20 @@ spec: mountPath: /etc/typha-ca/ca.crt readOnly: true {% endif %} - + - name: policysync + mountPath: /var/run/nodeagent +{% if calico_bpf_enabled %} + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional +{% endif %} + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true volumes: # Used by calico/node. - name: lib-modules @@ -375,6 +409,26 @@ spec: hostPath: path: "/etc/kubernetes/ssl/" {% endif %} +{% if calico_bpf_enabled %} + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate +{% endif %} + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: "{{ kubelet_flexvolumes_plugins_dir | default('/usr/libexec/kubernetes/kubelet-plugins/volume/exec') }}/nodeagent~uds" updateStrategy: rollingUpdate: maxUnavailable: {{ serial | default('20%') }} diff --git a/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/roles/network_plugin/calico/templates/calico-typha.yml.j2 index 143a1711e53470736f32b46f2cc9206d094ea431..f14f9b38d9925ba069d7e09bb95d0d2f6adfdc0f 100644 --- a/roles/network_plugin/calico/templates/calico-typha.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2 @@ -46,6 +46,10 @@ spec: k8s-app: calico-typha annotations: cluster-autoscaler.kubernetes.io/safe-to-evict: 'true' +{% if typha_prometheusmetricsenabled %} + prometheus.io/scrape: 'true' + prometheus.io/port: "{{ typha_prometheusmetricsport }}" +{% endif %} spec: nodeSelector: kubernetes.io/os: linux @@ -61,6 +65,9 @@ spec: # as a host-networked pod. serviceAccountName: calico-node priorityClassName: system-cluster-critical + # fsGroup allows using projected serviceaccount tokens as described here kubernetes/kubernetes#82573 + securityContext: + fsGroup: 65534 containers: - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }} name: calico-typha @@ -68,6 +75,11 @@ spec: - containerPort: 5473 name: calico-typha protocol: TCP + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true env: # Enable "info" logging by default. Can be set to "debug" to increase verbosity. - name: TYPHA_LOGSEVERITYSCREEN @@ -105,13 +117,14 @@ spec: name: cacert readOnly: true {% endif %} - # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked, +{% if typha_prometheusmetricsenabled %} + # Since Typha is host-networked, # this opens a port on the host, which may need to be secured. - #- name: TYPHA_PROMETHEUSMETRICSENABLED - # value: "true" - #- name: TYPHA_PROMETHEUSMETRICSPORT - # value: "9093" - + - name: TYPHA_PROMETHEUSMETRICSENABLED + value: "true" + - name: TYPHA_PROMETHEUSMETRICSPORT + value: "{{ typha_prometheusmetricsport }}" +{% endif %} # Needed for version >=3.7 when the 'host-local' ipam is used # Should never happen given templates/cni-calico.conflist.j2 # Configure route aggregation based on pod CIDR. diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 index d012d21f073017bd3d1a3dd2c37dd4fabe96b2dc..bc92f7009e60109e68f4f94813cc205f5231452a 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2 @@ -64,6 +64,12 @@ "capabilities": { "portMappings": true } + }, + { + "type":"bandwidth", + "capabilities": { + "bandwidth": true + } } ] } diff --git a/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..b2d1f407395dca9a1e331e841b6524a70ce0fd48 --- /dev/null +++ b/roles/network_plugin/calico/templates/kubernetes-services-endpoint.yml.j2 @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + namespace: kube-system + name: kubernetes-services-endpoint +data: +{% if calico_bpf_enabled %} +{% if loadbalancer_apiserver is defined %} + KUBERNETES_SERVICE_HOST: "{{ apiserver_loadbalancer_domain_name }}" + KUBERNETES_SERVICE_PORT: "{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}" +{%- elif use_localhost_as_kubeapi_loadbalancer|default(False)|bool %} + KUBERNETES_SERVICE_HOST: "127.0.0.1" + KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}" +{%- else %} + KUBERNETES_SERVICE_HOST: "{{ first_kube_master }}" + KUBERNETES_SERVICE_PORT: "{{ kube_apiserver_port }}" +{%- endif %} +{% endif %}