Fix resolvconf

Do not repeat options and nameservers in the dhclient hooks.
Do not prepend nameservers for dhclient but supersede and fail back
to the upstream_dns_resolvers then default_resolver. Fixes order of
nameservers placement, which is cluster DNS ip goes always first.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

docs/dns-stack.md

@@ -39,15 +39,17 @@ Later, the nameservers will be reconfigured to the DNS service IP that Kargo
 configures for K8s cluster.
 
 Also note, existing records will be purged from the `/etc/resolv.conf`,
-including base/head/cloud-init config files and those that come from dhclient.
+including resolvconf's base/head/cloud-init config files and those that come from dhclient.
 This is required for hostnet pods networking and for [kubelet to not exceed search domains
 limits](https://github.com/kubernetes/kubernetes/issues/9229).
 
-New search, nameserver records and options will be defined from the aforementioned vars:
-* Via resolvconf's head file, if resolvconf installed.
-* Via dhclient's DNS update hook.
-* Via cloud-init (CoreOS only).
-* Statically in the `/etc/resolv.conf`, if none of above is applicable.
+Instead, new domain, search, nameserver records and options will be defined from the
+aforementioned vars:
+* Superseded via dhclient's DNS update hook.
+* Generated via cloud-init (CoreOS only).
+* Statically defined in the `/etc/resolv.conf`, if none of above is applicable.
+* Resolvconf's head/base files are disabled from populating anything into the
+  `/etc/resolv.conf`.
 
 DNS configuration details
 -------------------------

roles/kubernetes/preinstall/tasks/dhclient-hooks.yml

 ---
-- name: Configure dhclient to prepend nameservers and supersede search/domain
+- name: Configure dhclient to supersede search/domain/nameservers
   blockinfile:
     block: |-
-      {% for item in [ supersede_domain, supersede_search, prepend_nameserver ] -%}
+      {% for item in [ supersede_domain, supersede_search, supersede_nameserver ] -%}
       {{ item }}
       {% endfor %}
     dest: "{{dhclientconffile}}"

roles/kubernetes/preinstall/tasks/set_resolv_facts.yml

@@ -64,18 +64,13 @@
     supersede_domain:
       supersede domain-name "{{ dns_domain }}";
 
-- name: decide on dns server IP
-  set_fact:
-    dns_server_real: >-
-      {%- if dns_early|bool -%}{{default_resolver}}{%- else -%}{{dns_server}}{%- endif -%}
-
 - name: pick dnsmasq cluster IP or default resolver
   set_fact:
     dnsmasq_server: |-
       {%- if skip_dnsmasq|bool and not dns_early|bool -%}
         {{ [ skydns_server ] + upstream_dns_servers|default([]) }}
       {%- elif dns_early|bool -%}
-        {{ [ dns_server_real ] + upstream_dns_servers|default([]) }}
+        {{ upstream_dns_servers|default([default_resolver]) }}
       {%- else -%}
         {{ [ dns_server ] }}
       {%- endif -%}
@@ -83,6 +78,6 @@
 - name: generate nameservers to resolvconf
   set_fact:
     nameserverentries:
-      nameserver {{( dnsmasq_server|default([]) + nameservers|default([])) | join(',nameserver ')}}
-    prepend_nameserver:
-      prepend domain-name-servers {{( dnsmasq_server|default([]) + nameservers|default([])) | join(', ') }};
+      nameserver {{( dnsmasq_server + nameservers|default([])) | join(',nameserver ')}}
+    supersede_nameserver:
+      supersede domain-name-servers {{( dnsmasq_server + nameservers|default([])) | join(', ') }};

roles/kubernetes/preinstall/templates/dhclient_dnsupdate.sh.j2

@@ -5,7 +5,7 @@
 #
 if [ $reason = "BOUND" ]; then
   if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
-    RESOLV_CONF=$(cat /etc/resolv.conf)
+    RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')
     OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
 
     printf "%b\n" "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf

roles/kubernetes/preinstall/templates/dhclient_dnsupdate_rh.sh.j2

@@ -5,7 +5,7 @@
 #
 zdnsupdate_config() {
   if [ -n "$new_domain_search" -o -n "$new_domain_name_servers" ]; then
-    RESOLV_CONF=$(cat /etc/resolv.conf)
+    RESOLV_CONF=$(cat /etc/resolv.conf | sed -r '/^options (timeout|attempts|ndots).*$/d')
     OPTIONS="options timeout:2\noptions attempts:2\noptions ndots:{{ ndots }}"
 
     echo -e "$RESOLV_CONF\n$OPTIONS" > /etc/resolv.conf