From 14cf3e138b4aa2e268c7328cf045be291ccbe548 Mon Sep 17 00:00:00 2001
From: Cristian Calin <6627509+cristicalin@users.noreply.github.com>
Date: Wed, 12 May 2021 15:22:17 +0300
Subject: [PATCH] Support Calico advertisement of MetalLB LoadBalancer IPs
(#7593)
* add initial MetalLB docs
* metallb allow disabling the deployment of the metallb speaker
* calico>=3.18 allow using calico to advertise service loadbalancer IPs
* Document the use of MetalLB and Calico
* clean MetalLB docs
---
README.md | 2 +
docs/metallb.md | 81 +++++++++++++++++++
.../sample/group_vars/k8s_cluster/addons.yml | 1 +
.../group_vars/k8s_cluster/k8s-net-calico.yml | 5 ++
roles/kubernetes-apps/metallb/README.md | 17 ----
.../kubernetes-apps/metallb/defaults/main.yml | 1 +
.../metallb/templates/metallb.yml.j2 | 10 +++
roles/network_plugin/calico/defaults/main.yml | 3 +
roles/network_plugin/calico/tasks/install.yml | 7 ++
9 files changed, 110 insertions(+), 17 deletions(-)
create mode 100644 docs/metallb.md
delete mode 100644 roles/kubernetes-apps/metallb/README.md
diff --git a/README.md b/README.md
index d89e19044..2d5dc6168 100644
--- a/README.md
+++ b/README.md
@@ -219,6 +219,8 @@ See also [Network checker](docs/netcheck.md).
- [nginx](https://kubernetes.github.io/ingress-nginx): the NGINX Ingress Controller.
+- [metallb](docs/metallb.md): the MetalLB bare-metal service LoadBalancer provider.
+
## Community docs and resources
- [kubernetes.io/docs/setup/production-environment/tools/kubespray/](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/)
diff --git a/docs/metallb.md b/docs/metallb.md
new file mode 100644
index 000000000..e8165d2e8
--- /dev/null
+++ b/docs/metallb.md
@@ -0,0 +1,81 @@
+# MetalLB
+
+MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation.
+It allows you to create Kubernetes services of type "LoadBalancer" in clusters that don't run on a cloud provider, and thus cannot simply hook into 3rd party products to provide load-balancers.
+The default operationg mode of MetalLB is in ["Layer2"](https://metallb.universe.tf/concepts/layer2/) but it can also operate in ["BGP"](https://metallb.universe.tf/concepts/bgp/) mode.
+
+## Install
+
+You have to explicitly enable the MetalLB extension and set an IP address range from which to allocate LoadBalancer IPs.
+
+```yaml
+metallb_enabled: true
+metallb_speaker_enabled: true
+metallb_ip_range:
+ - 10.5.0.0/16
+```
+
+By default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller:
+
+```yaml
+metallb_controller_tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Equal"
+ value: ""
+ effect: "NoSchedule"
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: "Equal"
+ value: ""
+ effect: "NoSchedule"
+```
+
+## BGP Mode
+
+When operating in BGP Mode MetalLB needs to have defined upstream peers:
+
+```yaml
+metallb_protocol: bgp
+metallb_ip_range:
+ - 10.5.0.0/16
+metallb_peers:
+ - peer_address: 192.0.2.1
+ peer_asn: 64512
+ my_asn: 4200000000
+ - peer_address: 192.0.2.2
+ peer_asn: 64513
+ my_asn: 4200000000
+```
+
+When using calico >= 3.18 you can replace MetalLB speaker by calico Service LoadBalancer IP advertisement.
+See [calico service IPs advertisement documentation](https://docs.projectcalico.org/archive/v3.18/networking/advertise-service-ips#advertise-service-load-balancer-ip-addresses).
+In this scenarion you should disable the MetalLB speaker and configure the `calico_advertise_service_loadbalancer_ips` to match your `metallb_ip_range`
+
+```yaml
+metallb_speaker_enabled: false
+metallb_ip_range:
+ - 10.5.0.0/16
+calico_advertise_service_loadbalancer_ips: "{{ metallb_ip_range }}"
+```
+
+If you have additional loadbalancer IP pool in `metallb_additional_address_pools`, ensure to add them to the list.
+
+```yaml
+metallb_speaker_enabled: false
+metallb_ip_range:
+ - 10.5.0.0/16
+metallb_additional_address_pools:
+ kube_service_pool_1:
+ ip_range:
+ - 10.6.0.0/16
+ protocol: "bgp"
+ auto_assign: false
+ kube_service_pool_2:
+ ip_range:
+ - 10.10.0.0/16
+ protocol: "bgp"
+ auto_assign: false
+calico_advertise_service_loadbalancer_ips:
+ - 10.5.0.0/16
+ - 10.6.0.0/16
+ - 10.10.0.0/16
+```
diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml
index cee77d634..1d08337a8 100644
--- a/inventory/sample/group_vars/k8s_cluster/addons.yml
+++ b/inventory/sample/group_vars/k8s_cluster/addons.yml
@@ -132,6 +132,7 @@ cert_manager_enabled: false
# MetalLB deployment
metallb_enabled: false
+metallb_speaker_enabled: true
# metallb_ip_range:
# - "10.5.0.50-10.5.0.99"
# metallb_speaker_nodeselector:
diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml
index 50215e4ca..84f24b414 100644
--- a/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-net-calico.yml
@@ -50,6 +50,11 @@
# - x.x.x.x/24
# - y.y.y.y/32
+# Adveritse Service LoadBalancer IPs
+# calico_advertise_service_loadbalancer_ips:
+# - x.x.x.x/24
+# - y.y.y.y/16
+
# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore)
# calico_datastore: "kdd"
diff --git a/roles/kubernetes-apps/metallb/README.md b/roles/kubernetes-apps/metallb/README.md
deleted file mode 100644
index 1456a6e8a..000000000
--- a/roles/kubernetes-apps/metallb/README.md
+++ /dev/null
@@ -1,17 +0,0 @@
-# Deploy MetalLB into Kubespray/Kubernetes
-
-MetalLB hooks into your Kubernetes cluster, and provides a network load-balancer implementation.
-In short, it allows you to create Kubernetes services of type "LoadBalancer" in clusters that
-don't run on a cloud provider, and thus cannot simply hook into paid products to provide load-balancers.
-This addon aims to automate [MetalLB in layer 2 mode](https://metallb.universe.tf/concepts/layer2/)
-or [MetalLB in BGP mode](https://metallb.universe.tf/concepts/bgp/).
-It deploys MetalLB into Kubernetes and sets up a layer 2 or BGP load-balancer.
-
-## Install
-
-In the default, MetalLB is not deployed into your Kubernetes cluster.
-You can override the defaults by copying the contents of roles/kubernetes-apps/metallb/defaults/main.yml
-to somewhere in inventory/mycluster/group_vars such as inventory/mycluster/groups_vars/k8s_cluster/addons.yml
-and updating metallb_enabled option to `true`.
-In addition you need to update metallb_ip_range option on the addons.yml at least for suiting your network
-environment, because MetalLB allocates external IP addresses from this metallb_ip_range option.
diff --git a/roles/kubernetes-apps/metallb/defaults/main.yml b/roles/kubernetes-apps/metallb/defaults/main.yml
index 06a509e93..b98106c41 100644
--- a/roles/kubernetes-apps/metallb/defaults/main.yml
+++ b/roles/kubernetes-apps/metallb/defaults/main.yml
@@ -6,6 +6,7 @@ metallb_port: "7472"
metallb_limits_cpu: "100m"
metallb_limits_mem: "100Mi"
metallb_peers: []
+metallb_speaker_enabled: true
metallb_speaker_nodeselector: {}
metallb_controller_nodeselector: {}
metallb_speaker_tolerations:
diff --git a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 b/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
index 29be2b1a3..5da5d9bfa 100644
--- a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
+++ b/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
@@ -47,6 +47,7 @@ spec:
- secret
- emptyDir
---
+{% if metallb_speaker_enabled %}
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
@@ -85,6 +86,7 @@ spec:
- configMap
- secret
- emptyDir
+{% endif %}
---
apiVersion: v1
kind: ServiceAccount
@@ -94,6 +96,7 @@ metadata:
name: controller
namespace: metallb-system
---
+{% if metallb_speaker_enabled %}
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -101,6 +104,7 @@ metadata:
app: metallb
name: speaker
namespace: metallb-system
+{% endif %}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
@@ -140,6 +144,7 @@ rules:
verbs:
- use
---
+{% if metallb_speaker_enabled %}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@@ -172,6 +177,7 @@ rules:
- podsecuritypolicies
verbs:
- use
+{% endif %}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
@@ -220,6 +226,7 @@ subjects:
name: controller
namespace: metallb-system
---
+{% if metallb_speaker_enabled %}
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
@@ -234,6 +241,7 @@ subjects:
- kind: ServiceAccount
name: speaker
namespace: metallb-system
+{% endif %}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
@@ -267,6 +275,7 @@ subjects:
- kind: ServiceAccount
name: speaker
---
+{% if metallb_speaker_enabled %}
apiVersion: apps/v1
kind: DaemonSet
metadata:
@@ -353,6 +362,7 @@ spec:
tolerations:
{{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
{% endif %}
+{% endif %}
---
apiVersion: apps/v1
kind: Deployment
diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml
index 24b28bb7c..b35416221 100644
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@@ -29,6 +29,9 @@ global_as_num: "64512"
# Advertise Service External IPs
calico_advertise_service_external_ips: []
+# Adveritse Service LoadBalancer IPs
+calico_advertise_service_loadbalancer_ips: []
+
# Limits for apps
calico_node_memory_limit: 500M
calico_node_cpu_limit: 300m
diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml
index 9237cb587..b696d6a95 100644
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@@ -188,6 +188,12 @@
with_items: "{{ calico_advertise_service_external_ips }}"
run_once: yes
+- name: Populate Service LoadBalancer IPs
+ set_fact:
+ _service_loadbalancer_ips: "{{ _service_loadbalancer_ips|default([]) + [ {'cidr': item} ] }}"
+ with_items: "{{ calico_advertise_service_loadbalancer_ips }}"
+ run_once: yes
+
- name: "Determine nodeToNodeMesh needed state"
set_fact:
nodeToNodeMeshEnabled: "false"
@@ -213,6 +219,7 @@
{% if not calico_no_global_as_num|default(false) %}"asNumber": {{ global_as_num }},{% endif %}
"nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} ,
{% if calico_advertise_cluster_ips|default(false) %}"serviceClusterIPs": [{"cidr": "{{ kube_service_addresses }}" }],{% endif %}
+ {% if calico_version is version('v3.18.0', '>') and calico_advertise_service_loadbalancer_ips|length > 0 %}"serviceLoadBalancerIPs": {{ _service_loadbalancer_ips }},{% endif %}
"serviceExternalIPs": {{ _service_external_ips|default([]) }} }}
changed_when: false
when:
--
GitLab