From ea7dcd46d7cb12c8bbc060fccfd234be70601138 Mon Sep 17 00:00:00 2001
From: Jeroen Rijken <Jeroen0494@users.noreply.github.com>
Date: Tue, 30 May 2023 20:37:49 +0200
Subject: [PATCH] Update MetalLB deployment, wait for resource. (#9995)
* Update MetalLB deployment, wait for resource.
Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>
* yml to yaml, add basic test for metallb
Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>
---------
Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>
---
docs/metallb.md | 73 +++++++----
.../kubernetes-apps/metallb/defaults/main.yml | 5 -
roles/kubernetes-apps/metallb/tasks/main.yml | 114 ++++++++++++------
.../{metallb.yml.j2 => metallb.yaml.j2} | 50 +++++---
.../packet_centos7-flannel-addons-ha.yml | 23 ++++
5 files changed, 181 insertions(+), 84 deletions(-)
rename roles/kubernetes-apps/metallb/templates/{metallb.yml.j2 => metallb.yaml.j2} (98%)
diff --git a/docs/metallb.md b/docs/metallb.md
index 5b6c05af4..4a577b084 100644
--- a/docs/metallb.md
+++ b/docs/metallb.md
@@ -26,15 +26,39 @@ By default only the MetalLB BGP speaker is allowed to run on control plane nodes
```yaml
metallb_config:
controller:
+ nodeselector:
+ kubernetes.io/os: linux
tolerations:
- - key: "node-role.kubernetes.io/master"
- operator: "Equal"
- value: ""
- effect: "NoSchedule"
- - key: "node-role.kubernetes.io/control-plane"
- operator: "Equal"
- value: ""
- effect: "NoSchedule"
+ - key: "node-role.kubernetes.io/master"
+ operator: "Equal"
+ value: ""
+ effect: "NoSchedule"
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: "Equal"
+ value: ""
+ effect: "NoSchedule"
+```
+
+If you'd like to set additional nodeSelector and tolerations values, you can do so in the following fasion:
+
+```yaml
+metallb_config:
+ controller:
+ nodeselector:
+ kubernetes.io/os: linux
+ tolerations:
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: "Equal"
+ value: ""
+ effect: "NoSchedule"
+ speaker:
+ nodeselector:
+ kubernetes.io/os: linux
+ tolerations:
+ - key: "node-role.kubernetes.io/control-plane"
+ operator: "Equal"
+ value: ""
+ effect: "NoSchedule"
```
## Pools
@@ -137,7 +161,6 @@ In this scenario you should disable the MetalLB speaker and configure the `calic
```yaml
metallb_speaker_enabled: false
-metallb_avoid_buggy_ips: true
metallb_config:
address_pools:
primary:
@@ -177,22 +200,22 @@ metallb_config:
vpn-only: "1234:1"
NO_ADVERTISE: "65535:65282"
metallb_peers:
- peer1:
- peer_address: 10.6.0.1
- peer_asn: 64512
- my_asn: 4200000000
- communities:
- - vpn-only
- address_pool:
- - pool1
- peer2:
- peer_address: 10.10.0.1
- peer_asn: 64513
- my_asn: 4200000000
- communities:
- - NO_ADVERTISE
- address_pool:
- - pool2
+ peer1:
+ peer_address: 10.6.0.1
+ peer_asn: 64512
+ my_asn: 4200000000
+ communities:
+ - vpn-only
+ address_pool:
+ - pool1
+ peer2:
+ peer_address: 10.10.0.1
+ peer_asn: 64513
+ my_asn: 4200000000
+ communities:
+ - NO_ADVERTISE
+ address_pool:
+ - pool2
calico_advertise_service_loadbalancer_ips:
- 10.5.0.0/16
- 10.6.0.0/16
diff --git a/roles/kubernetes-apps/metallb/defaults/main.yml b/roles/kubernetes-apps/metallb/defaults/main.yml
index dc96fdc7d..e9012dc10 100644
--- a/roles/kubernetes-apps/metallb/defaults/main.yml
+++ b/roles/kubernetes-apps/metallb/defaults/main.yml
@@ -1,10 +1,8 @@
---
metallb_enabled: false
metallb_log_level: info
-metallb_protocol: "layer2"
metallb_port: "7472"
metallb_memberlist_port: "7946"
-metallb_peers: []
metallb_speaker_enabled: "{{ metallb_enabled }}"
metallb_speaker_nodeselector:
kubernetes.io/os: "linux"
@@ -18,6 +16,3 @@ metallb_speaker_tolerations:
key: node-role.kubernetes.io/control-plane
operator: Exists
metallb_controller_tolerations: []
-metallb_pool_name: "loadbalanced"
-metallb_auto_assign: true
-metallb_avoid_buggy_ips: false
diff --git a/roles/kubernetes-apps/metallb/tasks/main.yml b/roles/kubernetes-apps/metallb/tasks/main.yml
index 50dc6c849..60fd6c87f 100644
--- a/roles/kubernetes-apps/metallb/tasks/main.yml
+++ b/roles/kubernetes-apps/metallb/tasks/main.yml
@@ -5,13 +5,6 @@
when:
- "kube_proxy_mode == 'ipvs' and not kube_proxy_strict_arp"
-- name: Kubernetes Apps | Check BGP peers for MetalLB
- fail:
- msg: "metallb_peers is mandatory when metallb_protocol is bgp and metallb_speaker_enabled"
- when:
- - metallb_config.layer3 is defined and metallb_speaker_enabled
- - metallb_config.metallb_peers is not defined or not metallb_config.metallb_peers
-
- name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore
fail:
msg: "'matallb_auto_assign' configuration variable is deprecated, please use 'metallb_auto_assign' instead"
@@ -36,46 +29,95 @@
- name: Kubernetes Apps | Lay Down MetalLB
become: true
template:
- src: "{{ item }}.j2"
- dest: "{{ kube_config_dir }}/{{ item }}"
+ src: "metallb.yaml.j2"
+ dest: "{{ kube_config_dir }}/metallb.yaml"
mode: 0644
- with_items: ["metallb.yml", "pools.yaml", "layer2.yaml", "layer3.yaml"]
- register: "rendering"
+ register: metallb_rendering
when:
- - "inventory_hostname == groups['kube_control_plane'][0]"
-
-- name: Kubernetes Apps | Create MetalLB resources and replace existing
- k8s:
- definition: "{{ lookup('template', 'metallb.yaml') }}"
-
-- name: Kubernetes Apps | Wait for MetalLB controller to be running
- k8s_info:
- kind: Deployment
- namespace: metallb-system
- name: controller
- wait: True
- wait_sleep: 10
- wait_timeout: 360
- wait_condition:
- status: "True"
- type: Available
- register: result
- until: result is not failed
+ - inventory_hostname == groups['kube_control_plane'][0]
- name: Kubernetes Apps | Install and configure MetalLB
kube:
name: "MetalLB"
kubectl: "{{ bin_dir }}/kubectl"
- filename: "{{ kube_config_dir }}/{{ item.item }}"
- state: "{{ item.changed | ternary('latest','present') }}"
+ filename: "{{ kube_config_dir }}/metallb.yaml"
+ state: "{{ metallb_rendering.changed | ternary('latest','present') }}"
+ wait: true
become: true
- with_items: "{{ rendering.results }}"
when:
- - "inventory_hostname == groups['kube_control_plane'][0]"
+ - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: Kubernetes Apps | Wait for MetalLB controller to be running
+ command: "{{ bin_dir }}/kubectl -n metallb-system wait --for=condition=ready pod -l app=metallb,component=controller"
+ become: true
+ when:
+ - inventory_hostname == groups['kube_control_plane'][0]
+
+- name: MetalLB | Address pools
+ block:
+ - name: MetalLB | Layout address pools template
+ ansible.builtin.template:
+ src: pools.yaml.j2
+ dest: "{{ kube_config_dir }}/pools.yaml"
+ mode: 0644
+ register: pools_rendering
+
+ - name: MetalLB | Create address pools configuration
+ kube:
+ name: "MetalLB"
+ kubectl: "{{ bin_dir }}/kubectl"
+ filename: "{{ kube_config_dir }}/pools.yaml"
+ state: "{{ pools_rendering.changed | ternary('latest','present') }}"
+ become: true
+ when:
+ - inventory_hostname == groups['kube_control_plane'][0]
+ - metallb_config.address_pools is defined
+
+- name: MetalLB | Layer2
+ block:
+ - name: MetalLB | Layout layer2 template
+ ansible.builtin.template:
+ src: layer2.yaml.j2
+ dest: "{{ kube_config_dir }}/layer2.yaml"
+ mode: 0644
+ register: layer2_rendering
+
+ - name: MetalLB | Create layer2 configuration
+ kube:
+ name: "MetalLB"
+ kubectl: "{{ bin_dir }}/kubectl"
+ filename: "{{ kube_config_dir }}/layer2.yaml"
+ state: "{{ layer2_rendering.changed | ternary('latest','present') }}"
+ become: true
+ when:
+ - inventory_hostname == groups['kube_control_plane'][0]
+ - metallb_config.layer2 is defined
+
+- name: MetalLB | Layer3
+ block:
+ - name: MetalLB | Layout layer3 template
+ ansible.builtin.template:
+ src: layer3.yaml.j2
+ dest: "{{ kube_config_dir }}/layer3.yaml"
+ mode: 0644
+ register: layer3_rendering
+
+ - name: MetalLB | Create layer3 configuration
+ kube:
+ name: "MetalLB"
+ kubectl: "{{ bin_dir }}/kubectl"
+ filename: "{{ kube_config_dir }}/layer3.yaml"
+ state: "{{ layer3_rendering.changed | ternary('latest','present') }}"
+ become: true
+ when:
+ - inventory_hostname == groups['kube_control_plane'][0]
+ - metallb_config.layer3 is defined
+
- name: Kubernetes Apps | Delete MetalLB ConfigMap
- k8s:
+ kube:
name: config
- kind: ConfigMap
+ kubectl: "{{ bin_dir }}/kubectl"
+ resource: ConfigMap
namespace: metallb-system
state: absent
diff --git a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2 b/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2
similarity index 98%
rename from roles/kubernetes-apps/metallb/templates/metallb.yml.j2
rename to roles/kubernetes-apps/metallb/templates/metallb.yaml.j2
index 57d8f2bed..eab386ff8 100644
--- a/roles/kubernetes-apps/metallb/templates/metallb.yml.j2
+++ b/roles/kubernetes-apps/metallb/templates/metallb.yaml.j2
@@ -1,3 +1,13 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+ labels:
+ pod-security.kubernetes.io/audit: privileged
+ pod-security.kubernetes.io/enforce: privileged
+ pod-security.kubernetes.io/warn: privileged
+ name: metallb-system
+
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
@@ -1703,8 +1713,8 @@ spec:
template:
metadata:
annotations:
- prometheus.io/port: "{{ metallb_port }}"
- prometheus.io/scrape: "true"
+ prometheus.io/port: '{{ metallb_port }}'
+ prometheus.io/scrape: 'true'
labels:
app: metallb
component: controller
@@ -1719,7 +1729,7 @@ spec:
value: memberlist
- name: METALLB_DEPLOYMENT
value: controller
- image: {{ metallb_controller_image_repo }}:{{ metallb_version }}
+ image: "{{ metallb_controller_image_repo }}:{{ metallb_version }}"
livenessProbe:
failureThreshold: 3
httpGet:
@@ -1755,14 +1765,15 @@ spec:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
-{% if metallb_config.controller.tolerations %}
+{% if metallb_config.controller is defined and metallb_config.controller.tolerations is defined %}
tolerations:
{{ metallb_config.controller.tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
-{% endif %}
-{% if metallb_controller_nodeselector %}
- nodeSelector:
- {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) }}
{%- endif %}
+ nodeSelector:
+ {{ metallb_controller_nodeselector | to_nice_yaml | indent(width=8) -}}
+ {% if metallb_config.controller is defined and metallb_config.controller.nodeselector is defined %}
+ {{ metallb_config.controller.nodeselector | to_nice_yaml | indent(width=8) -}}
+ {%- endif %}
securityContext:
fsGroup: 65534
runAsNonRoot: true
@@ -1793,8 +1804,8 @@ spec:
template:
metadata:
annotations:
- prometheus.io/port: "{{ metallb_port }}"
- prometheus.io/scrape: "true"
+ prometheus.io/port: '{{ metallb_port }}'
+ prometheus.io/scrape: 'true'
labels:
app: metallb
component: speaker
@@ -1823,7 +1834,7 @@ spec:
secretKeyRef:
key: secretkey
name: memberlist
- image: {{ metallb_speaker_image_repo }}:{{ metallb_version }}
+ image: "{{ metallb_speaker_image_repo }}:{{ metallb_version }}"
livenessProbe:
failureThreshold: 3
httpGet:
@@ -1860,16 +1871,19 @@ spec:
- ALL
readOnlyRootFilesystem: true
hostNetwork: true
-{% if metallb_speaker_nodeselector %}
nodeSelector:
- {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) }}
-{%- endif %}
+ {{ metallb_speaker_nodeselector | to_nice_yaml | indent(width=8) -}}
+ {% if metallb_config.speaker is defined and metallb_config.speaker.nodeselector is defined %}
+ {{ metallb_config.speaker.nodeselector | to_nice_yaml | indent(width=8) -}}
+ {%- endif %}
+
serviceAccountName: speaker
terminationGracePeriodSeconds: 2
-{% if metallb_speaker_tolerations %}
tolerations:
- {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) }}
-{% endif %}
+ {{ metallb_speaker_tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
+ {% if metallb_config.speaker is defined and metallb_config.speaker.tolerations is defined %}
+ {{ metallb_config.speaker.tolerations | to_nice_yaml(indent=2) | indent(width=8) -}}
+ {% endif %}
{% endif %}
---
@@ -2004,7 +2018,7 @@ webhooks:
clientConfig:
service:
name: webhook-service
- namespace: metallb-system
+ namespace: metallb-system
path: /validate-metallb-io-v1beta1-l2advertisement
failurePolicy: Fail
name: l2advertisementvalidationwebhook.metallb.io
diff --git a/tests/files/packet_centos7-flannel-addons-ha.yml b/tests/files/packet_centos7-flannel-addons-ha.yml
index 26d5bbddd..ef7cb1bfd 100644
--- a/tests/files/packet_centos7-flannel-addons-ha.yml
+++ b/tests/files/packet_centos7-flannel-addons-ha.yml
@@ -49,3 +49,26 @@ kube_vip_enabled: true
kube_vip_arp_enabled: true
kube_vip_controlplane_enabled: true
kube_vip_address: 192.168.1.100
+
+# MetalLB
+metallb_enabled: true
+metallb_speaker_enabled: true
+metallb_config:
+ address_pools:
+ primary:
+ ip_range:
+ - 192.0.1.0-192.0.1.254
+ auto_assign: true
+ pool1:
+ ip_range:
+ - 192.0.2.1-192.0.2.1
+ auto_assign: false
+ pool2:
+ ip_range:
+ - 192.0.2.2-192.0.2.2
+ auto_assign: false
+
+ layer2:
+ - primary
+ - pool1
+ - pool2
--
GitLab