Fix host DNS config 1) being edited too soon and 2) not working with NM (#8575)

Signed-off-by: Mac Chaffee <me@macchaffee.com>

cluster.yml

@@ -118,7 +118,8 @@
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
     - { role: kubernetes-apps, tags: apps }
 
-- hosts: k8s_cluster
+- name: Apply resolv.conf changes now that cluster DNS is up
+  hosts: k8s_cluster
   gather_facts: False
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"

roles/kubernetes/preinstall/defaults/main.yml

@@ -3,6 +3,7 @@
 ignore_assert_errors: false
 
 epel_enabled: false
+# Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
 dns_late: false
 
 common_required_pkgs:

roles/kubernetes/preinstall/handlers/main.yml

@@ -23,12 +23,11 @@
   command: /usr/bin/coreos-cloudinit --from-file {{ resolveconf_cloud_init_conf }}
   when: ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"]
 
-- name: Preinstall | update resolvconf for Fedora CoreOS
+- name: Preinstall | update resolvconf for networkmanager
   command: /bin/true
   notify:
     - Preinstall | reload NetworkManager
     - Preinstall | reload kubelet
-  when: is_fedora_coreos
 
 - name: Preinstall | reload NetworkManager
   service:

roles/kubernetes/preinstall/tasks/0040-set_facts.yml

@@ -67,6 +67,14 @@
 
   when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists
 
+- name: NetworkManager | Check if host has NetworkManager
+  # noqa 303 Should we use service_facts for this?
+  command: systemctl is-active --quiet NetworkManager.service
+  register: networkmanager_enabled
+  failed_when: false
+  changed_when: false
+  check_mode: false
+
 - name: check systemd-resolved
   # noqa 303 Should we use service_facts for this?
   command: systemctl is-active systemd-resolved
@@ -98,8 +106,7 @@
 
 - name: check if early DNS configuration stage
   set_fact:
-    dns_early: >-
-      {%- if kubelet_configured.stat.exists -%}false{%- else -%}true{%- endif -%}
+    dns_early: "{{ not kubelet_configured.stat.exists }}"
 
 - name: target resolv.conf files
   set_fact:
@@ -177,12 +184,24 @@
         {{ upstream_dns_servers|default([]) }}
       {%- endif -%}
 
-- name: generate nameservers to resolvconf
+# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout
+- name: generate nameservers for resolvconf, including cluster DNS
   set_fact:
-    nameserverentries:
-      nameserver {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',nameserver ') }}
+    nameserverentries: |-
+      {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',') }}
     supersede_nameserver:
       supersede domain-name-servers {{ ( coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }};
+  when: not dns_early or dns_late
+
+# This task should run instead of the above task when cluster/nodelocal DNS hasn't
+# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml)
+- name: generate nameservers for resolvconf, not including cluster DNS
+  set_fact:
+    nameserverentries: |-
+      {{ ( nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',') }}
+    supersede_nameserver:
+      supersede domain-name-servers {{ ( nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }};
+  when: dns_early and not dns_late
 
 - name: gather os specific variables
   include_vars: "{{ item }}"

roles/kubernetes/preinstall/tasks/0060-resolvconf.yml

@@ -7,9 +7,12 @@
   blockinfile:
     path: "{{ resolvconffile }}"
     block: |-
-      {% for item in [domainentry] + [searchentries] + nameserverentries.split(',') -%}
+      {% for item in [domainentry] + [searchentries] -%}
       {{ item }}
       {% endfor %}
+      {% for item in nameserverentries.split(',') %}
+      nameserver {{ item }}
+      {% endfor %}
       options ndots:{{ ndots }}
       options timeout:2
       options attempts:2

roles/kubernetes/preinstall/tasks/0062-networkmanager-unmanaged-devices.yml

 ---
-- name: NetworkManager | Check if host has NetworkManager
-  # noqa 303 Should we use service_facts for this?
-  command: systemctl is-active --quiet NetworkManager.service
-  register: nm_check
-  failed_when: false
-  changed_when: false
-  check_mode: false
-
 - name: NetworkManager | Ensure NetworkManager conf.d dir
   file:
     path: "/etc/NetworkManager/conf.d"
     state: directory
     recurse: yes
-  when: nm_check.rc == 0
 
 - name: NetworkManager | Prevent NetworkManager from managing Calico interfaces (cali*/tunl*/vxlan.calico)
   copy:
@@ -22,7 +13,6 @@
     dest: /etc/NetworkManager/conf.d/calico.conf
     mode: 0644
   when:
-    - nm_check.rc == 0
     - kube_network_plugin == "calico"
   notify: Preinstall | reload NetworkManager
 
@@ -35,5 +25,4 @@
       unmanaged-devices+=interface-name:kube-ipvs0;interface-name:nodelocaldns
     dest: /etc/NetworkManager/conf.d/k8s.conf
     mode: 0644
-  when: nm_check.rc == 0
   notify: Preinstall | reload NetworkManager

roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml

@@ -4,10 +4,10 @@
     path: /etc/NetworkManager/conf.d/dns.conf
     section: global-dns-domain-*
     option: servers
-    value: "{{ ( coredns_server + nameservers|d([]) + cloud_resolver|d([])) | unique | join(',') }}"
+    value: "{{ nameserverentries }}"
     mode: '0600'
     backup: yes
-  notify: Preinstall | update resolvconf for Fedora CoreOS
+  notify: Preinstall | update resolvconf for networkmanager
 
 - name: NetworkManager | Add DNS search to NM configuration
   ini_file:
@@ -17,7 +17,7 @@
     value: "{{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(',') }}"
     mode: '0600'
     backup: yes
-  notify: Preinstall | update resolvconf for Fedora CoreOS
+  notify: Preinstall | update resolvconf for networkmanager
 
 - name: NetworkManager | Add DNS options to NM configuration
   ini_file:
@@ -27,4 +27,4 @@
     value: "ndots:{{ ndots }};timeout:2;attempts:2;"
     mode: '0600'
     backup: yes
-  notify: Preinstall | update resolvconf for Fedora CoreOS
+  notify: Preinstall | update resolvconf for networkmanager

roles/kubernetes/preinstall/tasks/main.yml

@@ -25,7 +25,7 @@
     - dns_mode != 'none'
     - resolvconf_mode == 'host_resolvconf'
     - systemd_resolved_enabled.rc != 0
-    - not is_fedora_coreos
+    - networkmanager_enabled.rc != 0
   tags:
     - bootstrap-os
     - resolvconf
@@ -40,6 +40,8 @@
     - resolvconf
 
 - import_tasks: 0062-networkmanager-unmanaged-devices.yml
+  when:
+    - networkmanager_enabled.rc == 0
   tags:
     - bootstrap-os
 
@@ -47,7 +49,7 @@
   when:
     - dns_mode != 'none'
     - resolvconf_mode == 'host_resolvconf'
-    - is_fedora_coreos
+    - networkmanager_enabled.rc == 0
   tags:
     - bootstrap-os
     - resolvconf

scale.yml

@@ -99,10 +99,11 @@
     - { role: kubernetes/node-label, tags: node-label }
     - { role: network_plugin, tags: network }
 
-- hosts: k8s_cluster
+- name: Apply resolv.conf changes now that cluster DNS is up
+  hosts: k8s_cluster
   gather_facts: False
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults }
-    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }

upgrade-cluster.yml

@@ -155,10 +155,11 @@
     - { role: kubespray-defaults }
     - { role: kubernetes-apps, tags: apps }
 
-- hosts: k8s_cluster
+- name: Apply resolv.conf changes now that cluster DNS is up
+  hosts: k8s_cluster
   gather_facts: False
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   environment: "{{ proxy_disable_env }}"
   roles:
     - { role: kubespray-defaults }
-    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf }
+    - { role: kubernetes/preinstall, when: "dns_mode != 'none' and resolvconf_mode == 'host_resolvconf'", tags: resolvconf, dns_late: true }