From 9cb12cf2507087662c9a9835b413ebcf221c8bc9 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Tue, 28 Feb 2017 17:56:00 +0300
Subject: [PATCH] Add autoscalers for dnsmasq and kubedns
By default kubedns and dnsmasq scale when installed.
Dnsmasq is no longer a daemonset. It is now a deployment.
Kubedns is no longer a replicationcluster. It is now a deployment.
Minimum replicas is two (to enable rolling updates).
Reduced memory erquirements for dnsmasq and kubedns
---
roles/dnsmasq/defaults/main.yml | 6 ++-
roles/dnsmasq/tasks/main.yml | 22 +++-----
roles/dnsmasq/tasks/pre_upgrade.yml | 9 ++++
.../dnsmasq/templates/dnsmasq-autoscaler.yml | 50 +++++++++++++++++++
.../{dnsmasq-ds.yml => dnsmasq-deploy.yml} | 9 +++-
.../kubernetes-apps/ansible/defaults/main.yml | 5 +-
roles/kubernetes-apps/ansible/tasks/main.yml | 7 +--
.../ansible/templates/kubedns-autoscaler.yml | 49 ++++++++++++++++++
.../{kubedns-rc.yml => kubedns-deploy.yml} | 11 ++--
9 files changed, 141 insertions(+), 27 deletions(-)
create mode 100644 roles/dnsmasq/tasks/pre_upgrade.yml
create mode 100644 roles/dnsmasq/templates/dnsmasq-autoscaler.yml
rename roles/dnsmasq/templates/{dnsmasq-ds.yml => dnsmasq-deploy.yml} (88%)
create mode 100644 roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml
rename roles/kubernetes-apps/ansible/templates/{kubedns-rc.yml => kubedns-deploy.yml} (96%)
diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml
index d8ac8b34b..21d051ded 100644
--- a/roles/dnsmasq/defaults/main.yml
+++ b/roles/dnsmasq/defaults/main.yml
@@ -25,4 +25,8 @@ dnsmasq_image_tag: "{{ dnsmasq_version }}"
dns_cpu_limit: 100m
dns_memory_limit: 170Mi
dns_cpu_requests: 70m
-dns_memory_requests: 70Mi
+dns_memory_requests: 50Mi
+
+# Autoscaler parameters
+dnsmasq_nodes_per_replica: 10
+dnsmasq_min_replicas: 1
diff --git a/roles/dnsmasq/tasks/main.yml b/roles/dnsmasq/tasks/main.yml
index f8654a262..d6112fd6e 100644
--- a/roles/dnsmasq/tasks/main.yml
+++ b/roles/dnsmasq/tasks/main.yml
@@ -1,4 +1,6 @@
---
+- include: pre_upgrade.yml
+
- name: ensure dnsmasq.d directory exists
file:
path: /etc/dnsmasq.d
@@ -54,26 +56,15 @@
src: "{{item.file}}"
dest: "{{kube_config_dir}}/{{item.file}}"
with_items:
- - {file: dnsmasq-ds.yml, type: ds}
- - {file: dnsmasq-svc.yml, type: svc}
+ - {name: dnsmasq, file: dnsmasq-deploy.yml, type: deployment}
+ - {name: dnsmasq, file: dnsmasq-svc.yml, type: svc}
+ - {name: dnsmasq-autoscaler, file: dnsmasq-autoscaler.yml, type: deployment}
register: manifests
when: inventory_hostname == groups['kube-master'][0]
-#FIXME: remove manifests.changed condition if kubernetes/features#124 is implemented
-- name: Delete existing dnsmasq daemonset
- kube:
- name: dnsmasq
- namespace: "{{system_namespace}}"
- kubectl: "{{bin_dir}}/kubectl"
- resource: "ds"
- filename: "{{kube_config_dir}}/{{item.item.file}}"
- state: absent
- with_items: "{{ manifests.results }}"
- when: inventory_hostname == groups['kube-master'][0] and item.item.type == "ds" and (manifests.changed or dnsmasq_config.changed)
-
- name: Start Resources
kube:
- name: dnsmasq
+ name: "{{item.item.name}}"
namespace: "{{system_namespace}}"
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
@@ -88,3 +79,4 @@
port: 53
delay: 5
when: inventory_hostname == groups['kube-node'][0]
+
diff --git a/roles/dnsmasq/tasks/pre_upgrade.yml b/roles/dnsmasq/tasks/pre_upgrade.yml
new file mode 100644
index 000000000..9d1517580
--- /dev/null
+++ b/roles/dnsmasq/tasks/pre_upgrade.yml
@@ -0,0 +1,9 @@
+---
+- name: Delete legacy dnsmasq daemonset
+ kube:
+ name: dnsmasq
+ namespace: "{{system_namespace}}"
+ kubectl: "{{bin_dir}}/kubectl"
+ resource: "ds"
+ state: absent
+ when: inventory_hostname == groups['kube-master'][0]
diff --git a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml
new file mode 100644
index 000000000..ca65c2dab
--- /dev/null
+++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml
@@ -0,0 +1,50 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: dnsmasq-autoscaler
+ namespace: kube-system
+ labels:
+ k8s-app: dnsmasq-autoscaler
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ template:
+ metadata:
+ labels:
+ k8s-app: dnsmasq-autoscaler
+ annotations:
+ scheduler.alpha.kubernetes.io/critical-pod: ''
+ scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+ spec:
+ containers:
+ - name: autoscaler
+ image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.1
+ resources:
+ requests:
+ cpu: "20m"
+ memory: "10Mi"
+ command:
+ - /cluster-proportional-autoscaler
+ - --namespace=kube-system
+ - --configmap=dnsmasq-autoscaler
+ - --target=ReplicationController/dnsmasq
+ # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate.
+ # If using small nodes, "nodesPerReplica" should dominate.
+ - --default-params={"linear":{"nodesPerReplica":{{ dnsmasq_nodes_per_replica }},"preventSinglePointFailure":true}}
+ - --logtostderr=true
+ - --v={{ kube_log_level }}
+
diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-deploy.yml
similarity index 88%
rename from roles/dnsmasq/templates/dnsmasq-ds.yml
rename to roles/dnsmasq/templates/dnsmasq-deploy.yml
index adcbbeacb..c1bb614ec 100644
--- a/roles/dnsmasq/templates/dnsmasq-ds.yml
+++ b/roles/dnsmasq/templates/dnsmasq-deploy.yml
@@ -1,16 +1,22 @@
---
apiVersion: extensions/v1beta1
-kind: DaemonSet
+kind: Deployment
metadata:
name: dnsmasq
namespace: "{{system_namespace}}"
labels:
k8s-app: dnsmasq
+ kubernetes.io/cluster-service: "true"
spec:
+ replicas: {{ dnsmasq_min_replicas }}
+ selector:
+ matchLabels:
+ k8s-app: dnsmasq
template:
metadata:
labels:
k8s-app: dnsmasq
+ kubernetes.io/cluster-service: "true"
spec:
containers:
- name: dnsmasq
@@ -55,3 +61,4 @@ spec:
hostPath:
path: /etc/dnsmasq.d-available
dnsPolicy: Default # Don't use cluster DNS.
+
diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml
index a69c80676..925dd03b8 100644
--- a/roles/kubernetes-apps/ansible/defaults/main.yml
+++ b/roles/kubernetes-apps/ansible/defaults/main.yml
@@ -7,8 +7,9 @@ exechealthz_version: 1.1
dns_cpu_limit: 100m
dns_memory_limit: 170Mi
dns_cpu_requests: 70m
-dns_memory_requests: 70Mi
-dns_replicas: 1
+dns_memory_requests: 50Mi
+kubedns_min_replicas: 1
+kubedns_nodes_per_replica: 10
# Images
kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64"
diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml
index 04554e785..de38d28ff 100644
--- a/roles/kubernetes-apps/ansible/tasks/main.yml
+++ b/roles/kubernetes-apps/ansible/tasks/main.yml
@@ -13,15 +13,16 @@
src: "{{item.file}}"
dest: "{{kube_config_dir}}/{{item.file}}"
with_items:
- - {file: kubedns-rc.yml, type: rc}
- - {file: kubedns-svc.yml, type: svc}
+ - {name: kubedns, file: kubedns-deploy.yml, type: deployment}
+ - {name: kubedns, file: kubedns-svc.yml, type: svc}
+ - {name: kubedns-autoscaler, file: kubedns-autoscaler.yml, type: deployment}
register: manifests
when: dns_mode != 'none' and inventory_hostname == groups['kube-master'][0]
tags: dnsmasq
- name: Kubernetes Apps | Start Resources
kube:
- name: kubedns
+ name: "{{item.item.name}}"
namespace: "{{ system_namespace }}"
kubectl: "{{bin_dir}}/kubectl"
resource: "{{item.item.type}}"
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml
new file mode 100644
index 000000000..65dee527f
--- /dev/null
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-autoscaler.yml
@@ -0,0 +1,49 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: kubedns-autoscaler
+ namespace: kube-system
+ labels:
+ k8s-app: kubedns-autoscaler
+ kubernetes.io/cluster-service: "true"
+ addonmanager.kubernetes.io/mode: Reconcile
+spec:
+ template:
+ metadata:
+ labels:
+ k8s-app: kubedns-autoscaler
+ annotations:
+ scheduler.alpha.kubernetes.io/critical-pod: ''
+ scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+ spec:
+ containers:
+ - name: autoscaler
+ image: gcr.io/google_containers/cluster-proportional-autoscaler-amd64:1.1.1
+ resources:
+ requests:
+ cpu: "20m"
+ memory: "10Mi"
+ command:
+ - /cluster-proportional-autoscaler
+ - --namespace=kube-system
+ - --configmap=kubedns-autoscaler
+ # Should keep target in sync with cluster/addons/dns/kubedns-controller.yaml.base
+ - --target=replicationcontroller/kubedns
+ - --default-params={"linear":{"nodesPerReplica":{{ kubedns_nodes_per_replica }},"min":{{ kubedns_min_replicas }}}}
+ - --logtostderr=true
+ - --v=2
+
diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml
similarity index 96%
rename from roles/kubernetes-apps/ansible/templates/kubedns-rc.yml
rename to roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml
index 13607c705..1bae177d3 100644
--- a/roles/kubernetes-apps/ansible/templates/kubedns-rc.yml
+++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml
@@ -1,5 +1,5 @@
-apiVersion: v1
-kind: ReplicationController
+apiVersion: extensions/v1beta1
+kind: Deployment
metadata:
name: kubedns
namespace: {{ system_namespace }}
@@ -8,10 +8,11 @@ metadata:
version: v19
kubernetes.io/cluster-service: "true"
spec:
- replicas: {{ dns_replicas }}
+ replicas: {{ kubedns_min_replicas }}
selector:
- k8s-app: kubedns
- version: v19
+ matchLabels:
+ k8s-app: kubedns
+ version: v19
template:
metadata:
labels:
--
GitLab