Skip to content
Snippets Groups Projects
Commit a56d9de5 authored by Bogdan Dobrelya's avatar Bogdan Dobrelya Committed by Bogdan Dobrelya
Browse files

Systemd units, limits, and bin path fixes


* Add restart for weave service unit
* Reuse docker_bin_dir everythere
* Limit systemd managed docker containers by CPU/RAM. Do not configure native
  systemd limits due to the lack of consensus in the kernel community
  requires out-of-tree kernel patches.

Signed-off-by: default avatarBogdan Dobrelya <bdobrelia@mirantis.com>
parent 6139ee3a
No related branches found
No related tags found
No related merge requests found
Showing
with 110 additions and 21 deletions
...@@ -11,6 +11,13 @@ spec: ...@@ -11,6 +11,13 @@ spec:
- name: kube-scheduler - name: kube-scheduler
image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ kube_scheduler_cpu_limit }}
memory: {{ kube_scheduler_memory_limit }}
requests:
cpu: {{ kube_scheduler_cpu_requests }}
memory: {{ kube_scheduler_memory_requests }}
command: command:
- /hyperkube - /hyperkube
- scheduler - scheduler
......
...@@ -9,6 +9,18 @@ kube_proxy_mode: iptables ...@@ -9,6 +9,18 @@ kube_proxy_mode: iptables
# If using the pure iptables proxy, SNAT everything # If using the pure iptables proxy, SNAT everything
kube_proxy_masquerade_all: true kube_proxy_masquerade_all: true
# Limits for kube components and nginx load balancer app
kubelet_memory_limit: 512M
kubelet_cpu_limit: 100m
kube_proxy_memory_limit: 2000M
kube_proxy_cpu_limit: 500m
kube_proxy_memory_requests: 256M
kube_proxy_cpu_requests: 150m
nginx_memory_limit: 512M
nginx_cpu_limit: 300m
nginx_memory_requests: 64M
nginx_cpu_requests: 50m
# kube_api_runtime_config: # kube_api_runtime_config:
# - extensions/v1beta1/daemonsets=true # - extensions/v1beta1/daemonsets=true
# - extensions/v1beta1/deployments=true # - extensions/v1beta1/deployments=true
......
...@@ -39,7 +39,7 @@ DAEMON_USER=root ...@@ -39,7 +39,7 @@ DAEMON_USER=root
# #
do_start() do_start()
{ {
/usr/bin/docker rm -f kubelet &>/dev/null || true {{ docker_bin_dir }}/docker rm -f kubelet &>/dev/null || true
sleep 1 sleep 1
# Return # Return
# 0 if daemon has been started # 0 if daemon has been started
......
#!/bin/bash #!/bin/bash
/usr/bin/docker run --privileged \ {{ docker_bin_dir }}/docker run --privileged \
--net=host --pid=host --name=kubelet --restart=on-failure:5 \ --net=host --pid=host --name=kubelet --restart=on-failure:5 \
-v /etc/cni:/etc/cni:ro \ -v /etc/cni:/etc/cni:ro \
-v /opt/cni:/opt/cni:ro \ -v /opt/cni:/opt/cni:ro \
...@@ -9,6 +9,7 @@ ...@@ -9,6 +9,7 @@
-v {{ docker_daemon_graph }}:/var/lib/docker \ -v {{ docker_daemon_graph }}:/var/lib/docker \
-v /var/run:/var/run \ -v /var/run:/var/run \
-v /var/lib/kubelet:/var/lib/kubelet \ -v /var/lib/kubelet:/var/lib/kubelet \
--memory={{ kubelet_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ kubelet_cpu_limit|regex_replace('m', '') }} \
{{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag}} \
nsenter --target=1 --mount --wd=. -- \ nsenter --target=1 --mount --wd=. -- \
./hyperkube kubelet \ ./hyperkube kubelet \
......
...@@ -23,8 +23,8 @@ ExecStart={{ bin_dir }}/kubelet \ ...@@ -23,8 +23,8 @@ ExecStart={{ bin_dir }}/kubelet \
$DOCKER_SOCKET \ $DOCKER_SOCKET \
$KUBELET_NETWORK_PLUGIN \ $KUBELET_NETWORK_PLUGIN \
$KUBELET_CLOUDPROVIDER $KUBELET_CLOUDPROVIDER
ExecStartPre=-/usr/bin/docker rm -f kubelet ExecStartPre=-{{ docker_bin_dir }}/docker rm -f kubelet
ExecReload=/usr/bin/docker restart kubelet ExecReload={{ docker_bin_dir }}/docker restart kubelet
Restart=always Restart=always
RestartSec=10s RestartSec=10s
......
...@@ -11,6 +11,13 @@ spec: ...@@ -11,6 +11,13 @@ spec:
- name: kube-proxy - name: kube-proxy
image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ kube_proxy_cpu_limit }}
memory: {{ kube_proxy_memory_limit }}
requests:
cpu: {{ kube_proxy_cpu_requests }}
memory: {{ kube_proxy_memory_requests }}
command: command:
- /hyperkube - /hyperkube
- proxy - proxy
......
...@@ -11,6 +11,13 @@ spec: ...@@ -11,6 +11,13 @@ spec:
- name: nginx-proxy - name: nginx-proxy
image: {{ nginx_image_repo }}:{{ nginx_image_tag }} image: {{ nginx_image_repo }}:{{ nginx_image_tag }}
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ nginx_cpu_limit }}
memory: {{ nginx_memory_limit }}
requests:
cpu: {{ nginx_cpu_requests }}
memory: {{ nginx_memory_requests }}
securityContext: securityContext:
privileged: true privileged: true
volumeMounts: volumeMounts:
......
...@@ -19,3 +19,17 @@ global_as_num: "64512" ...@@ -19,3 +19,17 @@ global_as_num: "64512"
# not be specified in calico CNI config, so Calico will use built-in # not be specified in calico CNI config, so Calico will use built-in
# defaults. The value should be a number, not a string. # defaults. The value should be a number, not a string.
# calico_mtu: 1500 # calico_mtu: 1500
# Limits for apps
calico_rr_memory_limit: 1000M
calico_rr_cpu_limit: 300m
calico_rr_memory_requests: 500M
calico_rr_cpu_requests: 150m
calico_node_memory_limit: 500M
calico_node_cpu_limit: 300m
calico_node_memory_requests: 256M
calico_node_cpu_requests: 150m
calicoctl_memory_limit: 170M
calicoctl_cpu_limit: 100m
calicoctl_memory_requests: 70M
calicoctl_cpu_requests: 50m
...@@ -5,8 +5,8 @@ Requires=docker.service ...@@ -5,8 +5,8 @@ Requires=docker.service
[Service] [Service]
EnvironmentFile=/etc/calico/calico-rr.env EnvironmentFile=/etc/calico/calico-rr.env
ExecStartPre=-/usr/bin/docker rm -f calico-rr ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr
ExecStart=/usr/bin/docker run --net=host --privileged \ ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
--name=calico-rr \ --name=calico-rr \
-e IP=${IP} \ -e IP=${IP} \
-e IP6=${IP6} \ -e IP6=${IP6} \
...@@ -16,12 +16,13 @@ ExecStart=/usr/bin/docker run --net=host --privileged \ ...@@ -16,12 +16,13 @@ ExecStart=/usr/bin/docker run --net=host --privileged \
-e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
-v /var/log/calico-rr:/var/log/calico \ -v /var/log/calico-rr:/var/log/calico \
-v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
--memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \
{{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }}
Restart=always Restart=always
RestartSec=10s RestartSec=10s
ExecStop=-/usr/bin/docker stop calico-rr ExecStop=-{{ docker_bin_dir }}/docker stop calico-rr
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
...@@ -41,7 +41,7 @@ ...@@ -41,7 +41,7 @@
notify: restart calico-node notify: restart calico-node
- name: Calico | Copy cni plugins from hyperkube - name: Calico | Copy cni plugins from hyperkube
command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
register: cni_task_result register: cni_task_result
until: cni_task_result.rc == 0 until: cni_task_result.rc == 0
retries: 4 retries: 4
...@@ -50,7 +50,7 @@ ...@@ -50,7 +50,7 @@
tags: [hyperkube, upgrade] tags: [hyperkube, upgrade]
- name: Calico | Copy cni plugins from calico/cni container - name: Calico | Copy cni plugins from calico/cni container
command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'" command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
register: cni_task_result register: cni_task_result
until: cni_task_result.rc == 0 until: cni_task_result.rc == 0
retries: 4 retries: 4
......
...@@ -5,8 +5,8 @@ Requires=docker.service ...@@ -5,8 +5,8 @@ Requires=docker.service
[Service] [Service]
EnvironmentFile=/etc/calico/calico.env EnvironmentFile=/etc/calico/calico.env
ExecStartPre=-/usr/bin/docker rm -f calico-node ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-node
ExecStart=/usr/bin/docker run --net=host --privileged \ ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
--name=calico-node \ --name=calico-node \
-e HOSTNAME=${CALICO_HOSTNAME} \ -e HOSTNAME=${CALICO_HOSTNAME} \
-e IP=${CALICO_IP} \ -e IP=${CALICO_IP} \
...@@ -24,12 +24,13 @@ ExecStart=/usr/bin/docker run --net=host --privileged \ ...@@ -24,12 +24,13 @@ ExecStart=/usr/bin/docker run --net=host --privileged \
-v /lib/modules:/lib/modules \ -v /lib/modules:/lib/modules \
-v /var/run/calico:/var/run/calico \ -v /var/run/calico:/var/run/calico \
-v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
--memory={{ calico_node_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_node_cpu_limit|regex_replace('m', '') }} \
{{ calico_node_image_repo }}:{{ calico_node_image_tag }} {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
Restart=always Restart=always
RestartSec=10s RestartSec=10s
ExecStop=-/usr/bin/docker stop calico-node ExecStop=-{{ docker_bin_dir }}/docker stop calico-node
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
#!/bin/bash #!/bin/bash
/usr/bin/docker run -i --privileged --rm \ {{ docker_bin_dir }}/docker run -i --privileged --rm \
--net=host --pid=host \ --net=host --pid=host \
-e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ -e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \
-e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \ -e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \
-e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \ -e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \
-e ETCD_KEY_FILE=/etc/calico/certs/key.pem \ -e ETCD_KEY_FILE=/etc/calico/certs/key.pem \
-v /usr/bin/docker:/usr/bin/docker \ -v {{ docker_bin_dir }}/docker:{{ docker_bin_dir }}/docker \
-v /var/run/docker.sock:/var/run/docker.sock \ -v /var/run/docker.sock:/var/run/docker.sock \
-v /var/run/calico:/var/run/calico \ -v /var/run/calico:/var/run/calico \
-v /etc/calico/certs:/etc/calico/certs:ro \ -v /etc/calico/certs:/etc/calico/certs:ro \
--memory={{ calicoctl_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calicoctl_cpu_limit|regex_replace('m', '') }} \
{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \ {{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
$@ $@
...@@ -13,3 +13,13 @@ canal_log_level: "info" ...@@ -13,3 +13,13 @@ canal_log_level: "info"
# Etcd SSL dirs # Etcd SSL dirs
canal_cert_dir: /etc/canal/certs canal_cert_dir: /etc/canal/certs
etcd_cert_dir: /etc/ssl/etcd/ssl etcd_cert_dir: /etc/ssl/etcd/ssl
# Limits for apps
calico_node_memory_limit: 500M
calico_node_cpu_limit: 200m
calico_node_memory_requests: 256M
calico_node_cpu_requests: 100m
flannel_memory_limit: 500M
flannel_cpu_limit: 200m
flannel_memory_requests: 256M
flannel_cpu_requests: 100m
...@@ -43,7 +43,7 @@ ...@@ -43,7 +43,7 @@
dest: "{{kube_config_dir}}/canal-node.yaml" dest: "{{kube_config_dir}}/canal-node.yaml"
- name: Canal | Copy cni plugins from hyperkube - name: Canal | Copy cni plugins from hyperkube
command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/" command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -a /opt/cni/bin/ /cnibindir/"
register: cni_task_result register: cni_task_result
until: cni_task_result.rc == 0 until: cni_task_result.rc == 0
retries: 4 retries: 4
...@@ -52,7 +52,7 @@ ...@@ -52,7 +52,7 @@
tags: [hyperkube, upgrade] tags: [hyperkube, upgrade]
- name: Canal | Copy cni plugins from calico/cni - name: Canal | Copy cni plugins from calico/cni
command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'" command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp -a /opt/cni/bin/* /cnibindir/'"
register: cni_task_result register: cni_task_result
until: cni_task_result.rc == 0 until: cni_task_result.rc == 0
retries: 4 retries: 4
......
...@@ -49,6 +49,13 @@ spec: ...@@ -49,6 +49,13 @@ spec:
- name: flannel - name: flannel
image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}" image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ flannel_cpu_limit }}
memory: {{ flannel_memory_limit }}
requests:
cpu: {{ flannel_cpu_requests }}
memory: {{ flannel_memory_requests }}
env: env:
# Cluster name # Cluster name
- name: CLUSTER_NAME - name: CLUSTER_NAME
...@@ -119,6 +126,13 @@ spec: ...@@ -119,6 +126,13 @@ spec:
- name: calico-node - name: calico-node
image: "{{ calico_node_image_repo }}:{{ calico_node_image_tag }}" image: "{{ calico_node_image_repo }}:{{ calico_node_image_tag }}"
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ calico_node_cpu_limit }}
memory: {{ calico_node_memory_limit }}
requests:
cpu: {{ calico_node_cpu_requests }}
memory: {{ calico_node_memory_requests }}
env: env:
# The location of the etcd cluster. # The location of the etcd cluster.
- name: ETCD_ENDPOINTS - name: ETCD_ENDPOINTS
......
--- ---
- name: Cloud | Copy cni plugins from hyperkube - name: Cloud | Copy cni plugins from hyperkube
command: "/usr/bin/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/" command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
register: cni_task_result register: cni_task_result
until: cni_task_result.rc == 0 until: cni_task_result.rc == 0
retries: 4 retries: 4
......
...@@ -10,3 +10,9 @@ flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address ...@@ -10,3 +10,9 @@ flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address
# You can choose what type of flannel backend to use # You can choose what type of flannel backend to use
# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md # please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
flannel_backend_type: "vxlan" flannel_backend_type: "vxlan"
# Limits for apps
flannel_memory_limit: 500M
flannel_cpu_limit: 300m
flannel_memory_requests: 256M
flannel_cpu_requests: 150m
...@@ -32,7 +32,7 @@ ...@@ -32,7 +32,7 @@
pause: seconds=10 prompt="Waiting for docker restart" pause: seconds=10 prompt="Waiting for docker restart"
- name: Flannel | wait for docker - name: Flannel | wait for docker
command: /usr/bin/docker images command: "{{ docker_bin_dir }}/docker images"
register: docker_ready register: docker_ready
retries: 10 retries: 10
delay: 5 delay: 5
......
...@@ -19,6 +19,13 @@ ...@@ -19,6 +19,13 @@
- name: "flannel-container" - name: "flannel-container"
image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}" image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
imagePullPolicy: {{ k8s_image_pull_policy }} imagePullPolicy: {{ k8s_image_pull_policy }}
resources:
limits:
cpu: {{ flannel_cpu_limit }}
memory: {{ flannel_memory_limit }}
requests:
cpu: {{ flannel_cpu_requests }}
memory: {{ flannel_memory_requests }}
command: command:
- "/bin/sh" - "/bin/sh"
- "-c" - "-c"
...@@ -26,9 +33,6 @@ ...@@ -26,9 +33,6 @@
ports: ports:
- hostPort: 10253 - hostPort: 10253
containerPort: 10253 containerPort: 10253
resources:
limits:
cpu: "100m"
volumeMounts: volumeMounts:
- name: "subnetenv" - name: "subnetenv"
mountPath: "/run/flannel" mountPath: "/run/flannel"
......
---
# Limits
weave_memory_limit: 500M
weave_cpu_limit: 300m
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment