Skip to content
Snippets Groups Projects
Commit 3e3ee0ae authored by Thomas Nys's avatar Thomas Nys Committed by Kubernetes Prow Robot
Browse files

Add support for running a nodelocal dns cache (#3861)

* Add support for running a nodelocal dns cache

After encountering dns issues in a cluster I was recently working on I
noticed Kubernetes 1.13 introduced support for running a nodelocal dns
cache.

I believe this can usefull for more people.

https://github.com/kubernetes/kubernetes/commit/73b548db06c5e293533344c5b6171e955eac9ff1
https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md

* Add requested changes

* Add additional requested changes + documentation

* Add requested changes after review

* Replace incorrect variable
parent 7b674e06
No related branches found
No related tags found
No related merge requests found
Showing
with 223 additions and 0 deletions
......@@ -84,6 +84,11 @@ leaves you with a non functional cluster.
``resolvconf_mode`` configures how Kubespray will setup DNS for ``hostNetwork: true`` PODs and non-k8s containers.
There are three modes available:
## Nodelocal DNS cache
Setting ``enable_nodelocaldns`` to ``true`` will make pods reach out to the dns (core-dns) caching agent running on the same node, thereby avoiding iptables DNAT rules and connection tracking. The local caching agent will query kube-dns / core-dns (depending on what main DNS plugin is configured in your cluster) for cache misses of cluster hostnames(cluster.local suffix by default).
More information on the rationale behind this implementation can be found [here](https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/0030-nodelocal-dns-cache.md).
#### docker_dns (default)
This sets up the docker daemon with additional --dns/--dns-search/--dns-opt flags.
......
......@@ -114,6 +114,8 @@ ndots: 2
dns_mode: coredns
# Set manual server if using a custom cluster DNS server
#manual_dns_server: 10.x.x.x
# Enable nodelocal dns cache
enable_nodelocaldns: False
# Can be docker_dns, host_resolvconf or none
resolvconf_mode: docker_dns
......
......@@ -184,6 +184,10 @@ coredns_version: "1.2.6"
coredns_image_repo: "coredns/coredns"
coredns_image_tag: "{{ coredns_version }}"
nodelocaldns_version: "1.15.0"
nodelocaldns_image_repo: "k8s.gcr.io/k8s-dns-node-cache"
nodelocaldns_image_tag: "{{ nodelocaldns_version }}"
dnsmasq_nanny_image_repo: "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-{{ image_arch }}"
dnsmasq_nanny_image_tag: "{{ kubedns_version }}"
dnsmasq_sidecar_image_repo: "gcr.io/google_containers/k8s-dns-sidecar-{{ image_arch }}"
......@@ -490,6 +494,15 @@ downloads:
groups:
- kube-node
nodelocaldns:
enabled: "{{ enable_nodelocaldns == True }}"
container: true
repo: "{{ nodelocaldns_image_repo }}"
tag: "{{ nodelocaldns_image_tag }}"
sha256: "{{ nodelocaldns_digest_checksum|default(None) }}"
groups:
- kube-node
dnsmasq_nanny:
enabled: "{{ dns_mode in ['kubedns', 'dnsmasq_kubedns'] }}"
container: true
......
......@@ -8,6 +8,12 @@ dns_nodes_per_replica: 10
dns_cores_per_replica: 20
dns_prevent_single_point_failure: "{{ 'true' if dns_min_replicas|int > 1 else 'false' }}"
# nodelocaldns
nodelocaldns_cpu_requests: 100m
nodelocaldns_ip: 169.254.25.10
nodelocaldns_memory_limit: 170Mi
nodelocaldnsdns_memory_requests: 70Mi
# Netchecker
deploy_netchecker: false
netchecker_port: 31081
......
......@@ -13,6 +13,19 @@
tags:
- upgrade
- name: Kubernetes Apps | Delete old nodelocalDNS resources
kube:
name: "nodelocaldns"
namespace: "kube-system"
kubectl: "{{ bin_dir }}/kubectl"
resource: "{{ item }}"
state: absent
with_items:
- 'deamonset'
- 'configmap'
tags:
- upgrade
- name: Kubernetes Apps | Delete kubeadm CoreDNS
kube:
name: "coredns"
......
......@@ -20,6 +20,7 @@
- dnsmasq
- coredns
- kubedns
- nodelocaldns
- name: Kubernetes Apps | CoreDNS
import_tasks: "tasks/coredns.yml"
......@@ -29,6 +30,14 @@
tags:
- coredns
- name: Kubernetes Apps | nodelocalDNS
import_tasks: "tasks/nodelocaldns.yml"
when:
- enable_nodelocaldns == True
- inventory_hostname == groups['kube-master'] | first
tags:
- nodelocaldns
- name: Kubernetes Apps | KubeDNS
import_tasks: "tasks/kubedns.yml"
when:
......@@ -49,6 +58,7 @@
- "{{ kubedns_manifests.results | default({}) }}"
- "{{ coredns_manifests.results | default({}) }}"
- "{{ coredns_secondary_manifests.results | default({}) }}"
- "{{ nodelocaldns_manifests.results | default({}) }}"
when:
- dns_mode != 'none'
- inventory_hostname == groups['kube-master'][0]
......@@ -61,6 +71,7 @@
- dnsmasq
- coredns
- kubedns
- nodelocaldns
loop_control:
label: "{{ item.item.file }}"
......
---
- name: Kubernetes Apps | Lay Down nodelocaldns Template
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir }}/{{ item.file }}"
with_items:
- { name: nodelocaldns, file: nodelocaldns-config.yml, type: configmap }
- { name: nodelocaldns, file: nodelocaldns-sa.yml, type: sa }
- { name: nodelocaldns, file: nodelocaldns-deamonset.yml, type: daemonset }
register: nodelocaldns_manifests
vars:
clusterIP: "{{ skydns_server }}"
secondaryclusterIP: "{{ skydns_server_secondary }}"
when:
- enable_nodelocaldns == True
- inventory_hostname == groups['kube-master'] | first
tags:
- nodelocaldns
apiVersion: v1
kind: ConfigMap
metadata:
name: nodelocaldns
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: EnsureExists
data:
Corefile: |
{{ dns_domain }}:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if secondaryclusterIP is defined and dns_mode == 'coredns_dual' %}
forward . {{ clusterIP }} {{ secondaryclusterIP }} {
{% else %}
forward . {{ clusterIP }} {
{% endif %}
force_tcp
}
prometheus :9253
health {{ nodelocaldns_ip }}:8080
}
in-addr.arpa:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if secondaryclusterIP is defined %}
forward . {{ clusterIP }} {{ secondaryclusterIP }} {
{% else %}
forward . {{ clusterIP }} {
{% endif %}
force_tcp
}
prometheus :9253
}
ip6.arpa:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if secondaryclusterIP is defined %}
forward . {{ clusterIP }} {{ secondaryclusterIP }} {
{% else %}
forward . {{ clusterIP }} {
{% endif %}
force_tcp
}
prometheus :9253
}
.:53 {
errors
cache 30
reload
loop
bind {{ nodelocaldns_ip }}
{% if resolvconf_mode == 'host_resolvconf' and upstream_dns_servers is defined and upstream_dns_servers|length > 0 %}
forward . {{ upstream_dns_servers|join(' ') }} {
{% else %}
forward . /etc/resolv.conf {
{% endif %}
force_tcp
}
prometheus :9253
}
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nodelocaldns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
k8s-app: nodelocaldns
template:
metadata:
labels:
k8s-app: nodelocaldns
spec:
{% if kube_version is version('v1.11.1', '>=') %}
priorityClassName: system-cluster-critical
{% endif %}
serviceAccountName: nodelocaldns
hostNetwork: true
dnsPolicy: Default # Don't use cluster DNS.
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
- key: "CriticalAddonsOnly"
operator: "Exists"
containers:
- name: node-cache
image: "{{ nodelocaldns_image_repo }}:{{ nodelocaldns_image_tag }}"
resources:
limits:
memory: {{ nodelocaldns_memory_limit }}
requests:
cpu: {{ nodelocaldns_cpu_requests }}
memory: {{ nodelocaldnsdns_memory_requests }}
args: [ "-localip", "{{ nodelocaldns_ip }}", "-conf", "/etc/coredns/Corefile" ]
securityContext:
privileged: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9253
name: metrics
protocol: TCP
livenessProbe:
httpGet:
host: {{ nodelocaldns_ip }}
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 3
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
volumes:
- name: config-volume
configMap:
name: nodelocaldns
items:
- key: Corefile
path: Corefile
terminationGracePeriodSeconds: 30
apiVersion: v1
kind: ServiceAccount
metadata:
name: nodelocaldns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
......@@ -46,6 +46,9 @@ ndots: 2
# Can be dnsmasq_kubedns, kubedns, manual or none
dns_mode: coredns
# Enable nodelocal dns cache
enable_nodelocaldns: False
# Should be set to a cluster IP if using a custom cluster DNS
# manual_dns_server: 10.x.x.x
......
......@@ -20,3 +20,4 @@ cert_manager_enabled: true
metrics_server_enabled: true
kube_token_auth: true
kube_basic_auth: true
enable_nodelocaldns: true
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment