From fa1d222eee41318758289baacbaff0777d76aad2 Mon Sep 17 00:00:00 2001
From: Alessio Greggi <ale_grey_91@hotmail.it>
Date: Mon, 2 May 2022 20:03:15 +0200
Subject: [PATCH] add support for `EventRateLimit` plugin configuration (#8711)
* feat: add support for EventRateLimit admission plugin
* docs: add documentation about admission_control_config_file and EventRateLimit configuration
---
docs/vars.md | 40 +++++++++++++++++++
.../control-plane/defaults/main/main.yml | 13 ++++++
.../control-plane/tasks/kubeadm-setup.yml | 24 +++++++++++
.../admission-controls.v1beta2.yaml.j2 | 9 +++++
.../templates/eventratelimit.v1beta2.yaml.j2 | 11 +++++
.../templates/kubeadm-config.v1beta2.yaml.j2 | 10 +++++
roles/kubernetes/control-plane/vars/main.yaml | 3 ++
7 files changed, 110 insertions(+)
create mode 100644 roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2
create mode 100644 roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2
create mode 100644 roles/kubernetes/control-plane/vars/main.yaml
diff --git a/docs/vars.md b/docs/vars.md
index 00d35e48a..304163568 100644
--- a/docs/vars.md
+++ b/docs/vars.md
@@ -57,36 +57,55 @@ Kubernetes needs some parameters in order to get deployed. These are the
following default cluster parameters:
* *cluster_name* - Name of cluster (default is cluster.local)
+
* *container_manager* - Container Runtime to install in the nodes (default is containerd)
+
* *image_command_tool* - Tool used to pull images (default depends on `container_manager`
and is `nerdctl` for `containerd`, `crictl` for `crio`, `docker` for `docker`)
+
* *image_command_tool_on_localhost* - Tool used to pull images on localhost
(default is equal to `image_command_tool`)
+
* *dns_domain* - Name of cluster DNS domain (default is cluster.local)
+
* *kube_network_plugin* - Plugin to use for container networking
+
* *kube_service_addresses* - Subnet for cluster IPs (default is
10.233.0.0/18). Must not overlap with kube_pods_subnet
+
* *kube_pods_subnet* - Subnet for Pod IPs (default is 10.233.64.0/18). Must not
overlap with kube_service_addresses.
+
* *kube_network_node_prefix* - Subnet allocated per-node for pod IPs. Remaining
bits in kube_pods_subnet dictates how many kube_nodes can be in cluster. Setting this > 25 will
raise an assertion in playbooks if the `kubelet_max_pods` var also isn't adjusted accordingly
(assertion not applicable to calico which doesn't use this as a hard limit, see
[Calico IP block sizes](https://docs.projectcalico.org/reference/resources/ippool#block-sizes).
+
* *enable_dual_stack_networks* - Setting this to true will provision both IPv4 and IPv6 networking for pods and services.
+
* *kube_service_addresses_ipv6* - Subnet for cluster IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1000/116``). Must not overlap with ``kube_pods_subnet_ipv6``.
+
* *kube_pods_subnet_ipv6* - Subnet for Pod IPv6 IPs (default is ``fd85:ee78:d8a6:8607::1:0000/112``). Must not overlap with ``kube_service_addresses_ipv6``.
+
* *kube_network_node_prefix_ipv6* - Subnet allocated per-node for pod IPv6 IPs. Remaining bits in ``kube_pods_subnet_ipv6`` dictates how many kube_nodes can be in cluster.
+
* *skydns_server* - Cluster IP for DNS (default is 10.233.0.3)
+
* *skydns_server_secondary* - Secondary Cluster IP for CoreDNS used with coredns_dual deployment (default is 10.233.0.4)
+
* *enable_coredns_k8s_external* - If enabled, it configures the [k8s_external plugin](https://coredns.io/plugins/k8s_external/)
on the CoreDNS service.
+
* *coredns_k8s_external_zone* - Zone that will be used when CoreDNS k8s_external plugin is enabled
(default is k8s_external.local)
+
* *enable_coredns_k8s_endpoint_pod_names* - If enabled, it configures endpoint_pod_names option for kubernetes plugin.
on the CoreDNS service.
+
* *cloud_provider* - Enable extra Kubelet option if operating inside GCE or
OpenStack (default is unset)
+
* *kube_feature_gates* - A list of key=value pairs that describe feature gates for
alpha/experimental Kubernetes features. (defaults is `[]`).
Additionally, you can use also the following variables to individually customize your kubernetes components installation (they works exactly like `kube_feature_gates`):
@@ -95,8 +114,10 @@ following default cluster parameters:
* *kube_scheduler_feature_gates*
* *kube_proxy_feature_gates*
* *kubelet_feature_gates*
+
* *kubeadm_feature_gates* - A list of key=value pairs that describe feature gates for
alpha/experimental Kubeadm features. (defaults is `[]`)
+
* *authorization_modes* - A list of [authorization mode](
https://kubernetes.io/docs/admin/authorization/#using-flags-for-your-authorization-module)
that the cluster should be configured for. Defaults to `['Node', 'RBAC']`
@@ -106,6 +127,25 @@ following default cluster parameters:
require a service account and cluster role bindings. You can override this
setting by setting authorization_modes to `[]`.
+* *kube_apiserver_admission_control_config_file* - Enable configuration for `kube-apiserver` admission plugins.
+ Currently this variable allow you to configure the `EventRateLimit` admission plugin.
+
+ To configure the **EventRateLimit** plugin you have to define a data structure like this:
+
+```yml
+kube_apiserver_admission_event_rate_limits:
+ limit_1:
+ type: Namespace
+ qps: 50
+ burst: 100
+ cache_size: 2000
+ limit_2:
+ type: User
+ qps: 50
+ burst: 100
+ ...
+```
+
Note, if cloud providers have any use of the ``10.233.0.0/16``, like instances'
private addresses, make sure to pick another values for ``kube_service_addresses``
and ``kube_pods_subnet``, for example from the ``172.18.0.0/16``.
diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml
index ad7037029..227a53b09 100644
--- a/roles/kubernetes/control-plane/defaults/main/main.yml
+++ b/roles/kubernetes/control-plane/defaults/main/main.yml
@@ -89,6 +89,19 @@ kube_apiserver_pod_eviction_unreachable_timeout_seconds: "300"
# 1.10+ admission plugins
kube_apiserver_enable_admission_plugins: []
+# enable admission plugins configuration
+kube_apiserver_admission_control_config_file: false
+
+# data structure to configure EventRateLimit admission plugin
+# this should have the following structure:
+# kube_apiserver_admission_event_rate_limits:
+# <limit_name>:
+# type: <limit_type>
+# qps: <qps_value>
+# burst: <burst_value>
+# cache_size: <cache_size_value>
+kube_apiserver_admission_event_rate_limits: {}
+
# 1.10+ list of disabled admission plugins
kube_apiserver_disable_admission_plugins: []
diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
index 23f798d64..cdb22f318 100644
--- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml
@@ -83,6 +83,30 @@
dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
mode: 0640
+- name: kubeadm | Create directory to store admission control configurations
+ file:
+ path: "{{ kube_config_dir }}/admission-controls"
+ state: directory
+ mode: 0640
+ when: kube_apiserver_admission_control_config_file
+
+- name: kubeadm | Push admission control config file
+ template:
+ src: "admission-controls.{{ kubeadmConfig_api_version }}.yaml.j2"
+ dest: "{{ kube_config_dir }}/admission-controls/admission-controls.yaml"
+ mode: 0640
+ when: kube_apiserver_admission_control_config_file
+
+- name: kubeadm | Push admission control config files
+ template:
+ src: "{{ item|lower }}.{{ kubeadmConfig_api_version }}.yaml.j2"
+ dest: "{{ kube_config_dir }}/admission-controls/{{ item|lower }}.yaml"
+ mode: 0640
+ when:
+ - kube_apiserver_admission_control_config_file
+ - item in kube_apiserver_admission_plugins_needs_configuration
+ loop: "{{ kube_apiserver_enable_admission_plugins[0].split(',') }}"
+
- name: kubeadm | Check if apiserver.crt contains all needed SANs
shell: |
set -o pipefail
diff --git a/roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2
new file mode 100644
index 000000000..0bb4517c2
--- /dev/null
+++ b/roles/kubernetes/control-plane/templates/admission-controls.v1beta2.yaml.j2
@@ -0,0 +1,9 @@
+apiVersion: apiserver.config.k8s.io/v1
+kind: AdmissionConfiguration
+plugins:
+{% for plugin in kube_apiserver_enable_admission_plugins[0].split(',') %}
+{% if plugin in kube_apiserver_admission_plugins_needs_configuration %}
+- name: {{ plugin }}
+ path: {{ kube_config_dir }}/{{ plugin|lower }}.yaml
+{% endif %}
+{% endfor %}
diff --git a/roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2
new file mode 100644
index 000000000..0d7867070
--- /dev/null
+++ b/roles/kubernetes/control-plane/templates/eventratelimit.v1beta2.yaml.j2
@@ -0,0 +1,11 @@
+apiVersion: eventratelimit.admission.k8s.io/v1alpha1
+kind: Configuration
+limits:
+{% for limit in kube_apiserver_admission_event_rate_limits.values() %}
+- type: {{ limit.type }}
+ qps: {{ limit.qps }}
+ burst: {{ limit.burst }}
+{% if limit.cache_size is defined %}
+ cacheSize: {{ limit.cache_size }}
+{% endif %}
+{% endfor %}
diff --git a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2
index bf7868bd8..a43c549de 100644
--- a/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2
+++ b/roles/kubernetes/control-plane/templates/kubeadm-config.v1beta2.yaml.j2
@@ -126,6 +126,9 @@ apiServer:
{% if kube_apiserver_enable_admission_plugins|length > 0 %}
enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }}
{% endif %}
+{% if kube_apiserver_admission_control_config_file %}
+ admission-control-config-file: {{ kube_config_dir }}/admission-controls.yaml
+{% endif %}
{% if kube_apiserver_disable_admission_plugins|length > 0 %}
disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }}
{% endif %}
@@ -249,6 +252,13 @@ apiServer:
readOnly: false
{% endif %}
{% endif %}
+{% if kube_apiserver_admission_control_config_file %}
+ - name: admission-control-configs
+ hostPath: {{ kube_config_dir }}/admission-controls
+ mountPath: {{ kube_config_dir }}
+ readOnly: false
+ pathType: DirectoryOrCreate
+{% endif %}
{% for volume in apiserver_extra_volumes %}
- name: {{ volume.name }}
hostPath: {{ volume.hostPath }}
diff --git a/roles/kubernetes/control-plane/vars/main.yaml b/roles/kubernetes/control-plane/vars/main.yaml
new file mode 100644
index 000000000..57a39f784
--- /dev/null
+++ b/roles/kubernetes/control-plane/vars/main.yaml
@@ -0,0 +1,3 @@
+---
+# list of admission plugins that needs to be configured
+kube_apiserver_admission_plugins_needs_configuration: [EventRateLimit]
--
GitLab