From eeb7274d6590a87e64f88e0b47d3726150df2da1 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <matthew.mosesohn@gmail.com>
Date: Wed, 11 Oct 2017 19:47:42 +0100
Subject: [PATCH] Adjust memory reservation for master nodes (#1769)
---
roles/kubernetes/node/defaults/main.yml | 11 +++++++++--
.../kubernetes/node/templates/kubelet.kubeadm.env.j2 | 9 ++++++++-
.../kubernetes/node/templates/kubelet.standard.env.j2 | 11 +++++++++--
3 files changed, 26 insertions(+), 5 deletions(-)
diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml
index 1eb1046b9..4b6ed6b14 100644
--- a/roles/kubernetes/node/defaults/main.yml
+++ b/roles/kubernetes/node/defaults/main.yml
@@ -20,14 +20,21 @@ kubelet_enable_cri: true
kubelet_cgroups_per_qos: true
# Set to empty to avoid cgroup creation
kubelet_enforce_node_allocatable: "\"\""
+
# Set false to enable sharing a pid namespace between containers in a pod.
# Note that PID namespace sharing requires docker >= 1.13.1.
kubelet_disable_shared_pid: true
-# Limits for kube components and nginx load balancer app
-kubelet_memory_limit: 512M
+# Reserve this space for system resources
+kubelet_memory_limit: 256M
kubelet_cpu_limit: 100m
+# Reservation for master hosts
+kubelet_master_memory_limit: 512M
+kubelet_master_cpu_limit: 200m
+
kubelet_status_update_frequency: 10s
+
+# Limits for kube components and nginx load balancer app
kube_proxy_memory_limit: 2000M
kube_proxy_cpu_limit: 500m
kube_proxy_memory_requests: 64M
diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
index 722f567c2..817c8a34a 100644
--- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2
@@ -30,6 +30,13 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
{% endset %}
+{# Node reserved CPU/memory #}
+{% if is_kube_master|bool %}
+{% set kubelet_reserve %}--kube-reserved cpu={{ kubelet_master_cpu_limit }},memory={{ kubelet_master_memory_limit|regex_replace('Mi', 'M') }}{% endset %}
+{% else %}
+{% set kubelet_reserve %}--kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }}{% endset %}
+{% endif %}
+
{# DNS settings for kubelet #}
{% if dns_mode == 'kubedns' %}
{% set kubelet_args_cluster_dns %}--cluster-dns={{ skydns_server }}{% endset %}
@@ -41,7 +48,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% set kubelet_args_dns %}{{ kubelet_args_cluster_dns }} --cluster-domain={{ dns_domain }} --resolv-conf={{ kube_resolv_conf }}{% endset %}
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_reserve }}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %}
diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2
index b983aefa5..574a541ba 100644
--- a/roles/kubernetes/node/templates/kubelet.standard.env.j2
+++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2
@@ -12,7 +12,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{# Base kubelet args #}
{% set kubelet_args_base %}--pod-manifest-path={{ kube_manifest_dir }} \
--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \
---kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} \
--node-status-update-frequency={{ kubelet_status_update_frequency }} \
--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \
{% if kube_version | version_compare('v1.6', '>=') %}
@@ -36,6 +35,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{# Location of the apiserver #}
{% set kubelet_args_kubeconfig %}--kubeconfig={{ kube_config_dir}}/node-kubeconfig.yaml --require-kubeconfig{% endset %}
+
{% if standalone_kubelet|bool %}
{# We are on a master-only host. Make the master unschedulable in this case. #}
{% if kube_version | version_compare('v1.6', '>=') %}
@@ -47,6 +47,13 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% endif %}
{% endif %}
+{# Node reserved CPU/memory #}
+{% if is_kube_master|bool %}
+{% set kubelet_reserve %}--kube-reserved cpu={{ kubelet_master_cpu_limit }},memory={{ kubelet_master_memory_limit|regex_replace('Mi', 'M') }}{% endset %}
+{% else %}
+{% set kubelet_reserve %}--kube-reserved cpu={{ kubelet_cpu_limit }},memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }}{% endset %}
+{% endif %}
+
{# Kubelet node labels #}
{% if inventory_hostname in groups['kube-master'] %}
{% set node_labels %}--node-labels=node-role.kubernetes.io/master=true{% endset %}
@@ -57,7 +64,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
{% set node_labels %}--node-labels=node-role.kubernetes.io/node=true{% endset %}
{% endif %}
-KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {{ kubelet_args_dns }} {{ kubelet_args_kubeconfig }} {{ kubelet_reserve }} {{ node_labels }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}"
{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "flannel", "weave"] %}
KUBELET_NETWORK_PLUGIN="--network-plugin=cni --network-plugin-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"
{% elif kube_network_plugin is defined and kube_network_plugin == "weave" %}
--
GitLab