diff --git a/.gitlab-ci/packet.yml b/.gitlab-ci/packet.yml index de02cd98981ed262574a2b771507e5dfcd0b3308..41a2a4fe4054ed90bff9f9679b882e2857676a5d 100644 --- a/.gitlab-ci/packet.yml +++ b/.gitlab-ci/packet.yml @@ -2,6 +2,7 @@ .packet: extends: .testcases variables: + ANSIBLE_TIMEOUT: "120" CI_PLATFORM: packet SSH_USER: kubespray tags: diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index 062059206ead7bd90fec0cde8b7aa48df360ea29..90b47b86c5885d8596450c8647d37a82cd846b26 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -192,7 +192,7 @@ coredns_k8s_external_zone: k8s_external.local enable_coredns_k8s_endpoint_pod_names: false # Can be docker_dns, host_resolvconf or none -resolvconf_mode: docker_dns +resolvconf_mode: host_resolvconf # Deploy netchecker app to verify DNS resolve as an HTTP service deploy_netchecker: false # Ip address of the kubernetes skydns service diff --git a/roles/adduser/molecule/default/molecule.yml b/roles/adduser/molecule/default/molecule.yml index 4bb5dce3084b0911cf678ebe4ad5219db06aa131..80ebdad72702fb60ac542fa407c29f57bdcd440f 100644 --- a/roles/adduser/molecule/default/molecule.yml +++ b/roles/adduser/molecule/default/molecule.yml @@ -15,6 +15,10 @@ platforms: memory: 512 provisioner: name: ansible + config_options: + defaults: + callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint verifier: diff --git a/roles/bastion-ssh-config/molecule/default/molecule.yml b/roles/bastion-ssh-config/molecule/default/molecule.yml index 1d84db76c136b951c5cefeb094881404e4f0e83d..c0c29ae92e15b223791ef2282f00cff6e2a06379 100644 --- a/roles/bastion-ssh-config/molecule/default/molecule.yml +++ b/roles/bastion-ssh-config/molecule/default/molecule.yml @@ -15,6 +15,10 @@ platforms: memory: 512 provisioner: name: ansible + config_options: + defaults: + callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint inventory: diff --git a/roles/bootstrap-os/molecule/default/molecule.yml b/roles/bootstrap-os/molecule/default/molecule.yml index 081d929e8828342414e620e0d085f99936092461..0bb61eff65773b4a9a2ae8ae94b64909597ea4e7 100644 --- a/roles/bootstrap-os/molecule/default/molecule.yml +++ b/roles/bootstrap-os/molecule/default/molecule.yml @@ -35,6 +35,10 @@ platforms: memory: 512 provisioner: name: ansible + config_options: + defaults: + callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint inventory: diff --git a/roles/container-engine/containerd/molecule/default/molecule.yml b/roles/container-engine/containerd/molecule/default/molecule.yml index fb2cb9f9dcfa3be3e532797c920699e8cbab78ec..ebe3595b63c4e616f56709cd854daaa1165d2491 100644 --- a/roles/container-engine/containerd/molecule/default/molecule.yml +++ b/roles/container-engine/containerd/molecule/default/molecule.yml @@ -46,6 +46,7 @@ provisioner: config_options: defaults: callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint options: diff --git a/roles/container-engine/cri-o/molecule/default/molecule.yml b/roles/container-engine/cri-o/molecule/default/molecule.yml index 2ca990c1daffde279462f786cdba90cf73651d4a..56e6abd6102d09a59bd4f674e8e4565b74a7cb8f 100644 --- a/roles/container-engine/cri-o/molecule/default/molecule.yml +++ b/roles/container-engine/cri-o/molecule/default/molecule.yml @@ -38,6 +38,7 @@ provisioner: config_options: defaults: callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint options: diff --git a/roles/container-engine/docker/molecule/default/molecule.yml b/roles/container-engine/docker/molecule/default/molecule.yml index eaf6fae500eb37e04ce1b4192ca8e6bdf0aa6d2e..c30366215c60c5da5b7941f33cea763dbb884e23 100644 --- a/roles/container-engine/docker/molecule/default/molecule.yml +++ b/roles/container-engine/docker/molecule/default/molecule.yml @@ -18,6 +18,7 @@ provisioner: config_options: defaults: callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint options: diff --git a/roles/container-engine/gvisor/molecule/default/molecule.yml b/roles/container-engine/gvisor/molecule/default/molecule.yml index fc4ec0276bafc06cfa1cd540b21ef60d91db6f6e..657dc2862d6c2af3c501d4677a4df250ef77a698 100644 --- a/roles/container-engine/gvisor/molecule/default/molecule.yml +++ b/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -30,6 +30,7 @@ provisioner: config_options: defaults: callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint options: diff --git a/roles/container-engine/kata-containers/molecule/default/molecule.yml b/roles/container-engine/kata-containers/molecule/default/molecule.yml index 164a4708391a5367191baa934c6592cf49b38ed7..bb981205489714b5af273557e86e0f86a20c8c59 100644 --- a/roles/container-engine/kata-containers/molecule/default/molecule.yml +++ b/roles/container-engine/kata-containers/molecule/default/molecule.yml @@ -30,6 +30,7 @@ provisioner: config_options: defaults: callback_whitelist: profile_tasks + timeout: 120 lint: name: ansible-lint options: diff --git a/roles/kubernetes/preinstall/handlers/main.yml b/roles/kubernetes/preinstall/handlers/main.yml index 54a5f6a670817d6af3f0bf8260fa9e139c8dd247..667465b6fcb65d3c7f953cf7e49b50fc96b01498 100644 --- a/roles/kubernetes/preinstall/handlers/main.yml +++ b/roles/kubernetes/preinstall/handlers/main.yml @@ -9,6 +9,7 @@ - Preinstall | restart kube-controller-manager crio/containerd - Preinstall | restart kube-apiserver docker - Preinstall | restart kube-apiserver crio/containerd + - Preinstall | wait for the apiserver to be running when: not ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] and not is_fedora_coreos - name: Preinstall | update resolvconf for Flatcar Container Linux by Kinvolk @@ -101,6 +102,21 @@ - dns_mode != 'none' - resolvconf_mode == 'host_resolvconf' +# When running this as the last phase ensure we wait for kube-apiserver to come up +- name: Preinstall | wait for the apiserver to be running + uri: + url: "{{ kube_apiserver_endpoint }}/healthz" + validate_certs: no + register: result + until: result.status == 200 + retries: 60 + delay: 1 + when: + - dns_late + - inventory_hostname in groups['kube_control_plane'] + - dns_mode != 'none' + - resolvconf_mode == 'host_resolvconf' + - name: Preinstall | Restart systemd-resolved service: name: systemd-resolved diff --git a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml index cf6612b00ef8ed6747f3856d8e05d374ea9b016a..1cfd477774b103f5940cd8e02f3a197e23f48a4d 100644 --- a/roles/kubernetes/preinstall/tasks/0040-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0040-set_facts.yml @@ -34,6 +34,39 @@ changed_when: false check_mode: no +- name: check existence of /etc/resolvconf/resolv.conf.d + stat: + path: /etc/resolvconf/resolv.conf.d + get_attributes: no + get_checksum: no + get_mime: no + failed_when: false + register: resolvconfd_path + +- name: check status of /etc/resolv.conf + stat: + path: /etc/resolv.conf + follow: no + get_attributes: no + get_checksum: no + get_mime: no + failed_when: false + register: resolvconf_stat + +- block: + + - name: get content of /etc/resolv.conf + slurp: + src: /etc/resolv.conf + register: resolvconf_slurp + + - name: get currently configured nameservers + set_fact: + configured_nameservers: "{{ resolvconf_slurp.content | b64decode | regex_findall('\\s*nameserver\\s*(.*)') | ipaddr }}" + when: resolvconf_slurp.content is defined + + when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists + - name: check systemd-resolved # noqa 303 Should we use service_facts for this? command: systemctl is-active systemd-resolved @@ -45,7 +78,7 @@ - name: set dns facts set_fact: resolvconf: >- - {%- if resolvconf.rc == 0 -%}true{%- else -%}false{%- endif -%} + {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} bogus_domains: |- {% for d in [ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([]) -%} {{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./ @@ -147,7 +180,7 @@ - name: generate nameservers to resolvconf set_fact: nameserverentries: - nameserver {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(',nameserver ') }} + nameserver {{ ( ( [nodelocaldns_ip] if enable_nodelocaldns else []) + coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([]) + configured_nameservers|d([])) | unique | join(',nameserver ') }} supersede_nameserver: supersede domain-name-servers {{ ( coredns_server|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(', ') }}; diff --git a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml index a34d031b1e921d2d85eb1c74ff7698524d156417..65b55d7fb4796f22e528a7c685877037a84c3f9a 100644 --- a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml @@ -16,7 +16,7 @@ state: present insertbefore: BOF create: yes - backup: yes + backup: "{{ not resolvconf_stat.stat.islnk }}" marker: "# Ansible entries {mark}" mode: 0644 notify: Preinstall | propagate resolvconf to k8s components @@ -25,7 +25,7 @@ replace: path: "{{ item[0] }}" regexp: '^{{ item[1] }}[^#]*(?=# Ansible entries BEGIN)' - backup: yes + backup: "{{ not resolvconf_stat.stat.islnk }}" with_nested: - "{{ [resolvconffile, base|default(''), head|default('')] | difference(['']) }}" - [ 'search ', 'nameserver ', 'domain ', 'options ' ] @@ -36,13 +36,12 @@ path: "{{ item[0] }}" regexp: '(# Ansible entries END\n(?:(?!^{{ item[1] }}).*\n)*)(?:^{{ item[1] }}.*\n?)+' replace: '\1' - backup: yes + backup: "{{ not resolvconf_stat.stat.islnk }}" with_nested: - "{{ [resolvconffile, base|default(''), head|default('')] | difference(['']) }}" - [ 'search ', 'nameserver ', 'domain ', 'options ' ] notify: Preinstall | propagate resolvconf to k8s components - - name: get temporary resolveconf cloud init file content command: cat {{ resolvconffile }} register: cloud_config diff --git a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml b/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml index 28aed07407b3161fef9188677b1362f80b2b2973..f240d0fdf4e40324b209b2b8d32d65ae1312be3f 100644 --- a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml +++ b/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml @@ -22,7 +22,7 @@ owner: root mode: 0755 notify: Preinstall | propagate resolvconf to k8s components - when: ansible_os_family != "RedHat" + when: ansible_os_family not in [ "RedHat", "Suse" ] - name: Configure dhclient hooks for resolv.conf (RH-only) template: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 4d88c2889b695773d72ee36421ff5acda342db2e..12a28b9afbc36d2d53ff20ecc4564826aec06caf 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -106,7 +106,7 @@ nodelocaldns_secondary_skew_seconds: 5 manual_dns_server: "" # Can be docker_dns, host_resolvconf or none -resolvconf_mode: docker_dns +resolvconf_mode: host_resolvconf # Deploy netchecker app to verify DNS resolve as an HTTP service deploy_netchecker: false # Ip address of the kubernetes DNS service (called skydns for historical reasons) diff --git a/tests/files/packet_centos7-docker-weave-upgrade-ha.yml b/tests/files/packet_centos7-docker-weave-upgrade-ha.yml index 92d39306b040dab31bd1725e7e824a9f326353f4..265c97ddef76a9846df841208b5f74c33126864f 100644 --- a/tests/files/packet_centos7-docker-weave-upgrade-ha.yml +++ b/tests/files/packet_centos7-docker-weave-upgrade-ha.yml @@ -10,6 +10,7 @@ kubernetes_audit: true # Docker specific settings: container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns # Needed to upgrade from 1.16 to 1.17, otherwise upgrade is partial and bug followed upgrade_cluster_setup: true diff --git a/tests/files/packet_centos8-docker.yml b/tests/files/packet_centos8-docker.yml index fb7e9ba499435b19ebeae477c31963911b6c401b..3d53119cda4137e961266bde8840670c57c342da 100644 --- a/tests/files/packet_centos8-docker.yml +++ b/tests/files/packet_centos8-docker.yml @@ -10,3 +10,4 @@ calico_iptables_backend: "Auto" # Use docker container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns diff --git a/tests/files/packet_debian10-docker.yml b/tests/files/packet_debian10-docker.yml index a59371968802f872e52fd02d8e151d68e7549949..fc55e7f7f11a435903100cd33b84318f1c92fc06 100644 --- a/tests/files/packet_debian10-docker.yml +++ b/tests/files/packet_debian10-docker.yml @@ -6,3 +6,4 @@ mode: default # Use docker container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns diff --git a/tests/files/packet_debian11-docker.yml b/tests/files/packet_debian11-docker.yml index 3b93dd06e32a75fe78423aac9ef712be4aeed112..69ec8eb2537e671dac245ee83bc96ae6ab7942bf 100644 --- a/tests/files/packet_debian11-docker.yml +++ b/tests/files/packet_debian11-docker.yml @@ -6,3 +6,4 @@ mode: default # Use docker container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns diff --git a/tests/files/packet_fedora34-docker-weave.yml b/tests/files/packet_fedora34-docker-weave.yml index f9153538f58100f017d73ab838ec5e30af1b0bd7..2fdef725d3828c28dd8fde8ac2fd04456380ee3e 100644 --- a/tests/files/packet_fedora34-docker-weave.yml +++ b/tests/files/packet_fedora34-docker-weave.yml @@ -9,3 +9,4 @@ kube_network_plugin: weave # Docker specific settings: container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns diff --git a/tests/files/packet_ubuntu16-docker-weave-sep.yml b/tests/files/packet_ubuntu16-docker-weave-sep.yml index c49c6307e1ac4d5369a8c210bd1a53243ae74cd1..9b268e77e6ea91e984bbdaf60c36c8ef4bd6c206 100644 --- a/tests/files/packet_ubuntu16-docker-weave-sep.yml +++ b/tests/files/packet_ubuntu16-docker-weave-sep.yml @@ -10,6 +10,7 @@ auto_renew_certificates: true # Docker specific settings: container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns # Ubuntu 16 - docker containerd package available stopped at 1.4.6 docker_containerd_version: latest diff --git a/tests/files/packet_ubuntu18-docker.yml b/tests/files/packet_ubuntu18-docker.yml index 74f5fa34dc972b5d3591095cfca028dd4bbae020..548ff371e1a9e636dab8289c912bdfe4416dba35 100644 --- a/tests/files/packet_ubuntu18-docker.yml +++ b/tests/files/packet_ubuntu18-docker.yml @@ -7,3 +7,4 @@ vm_memory: 1600Mi # Use docker container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns diff --git a/tests/files/packet_ubuntu20-docker.yml b/tests/files/packet_ubuntu20-docker.yml index ca7c3c7e712c7bfe7fcbe22d18ba00e8a4e08cc7..4089a6605629910a57be1123686f0a2b00d756d9 100644 --- a/tests/files/packet_ubuntu20-docker.yml +++ b/tests/files/packet_ubuntu20-docker.yml @@ -14,3 +14,4 @@ enable_nodelocaldns: False # Use docker container_manager: docker etcd_deployment_type: docker +resolvconf_mode: docker_dns