Upgrade container-engine after draining (#5601)

* Run 'container-engine' after drain.

Move possibly disruptive role 'container-engine' to run after the node
is drained.

As that role have to be run on non-cluster nodes as well (etcd and
calico-rr), and those nodes are not drained, add play for that case.

* Check if api is up before upgrade.

If container engine is restarted in previous role, api controller can
take some time to start. This check ensures api is up before upgrade.

roles/kubernetes/master/tasks/kubeadm-upgrade.yml

 ---
+- name: kubeadm | Check api is up
+  uri:
+    url: "https://{{ ip | default(fallback_ips[inventory_hostname]) }}:6443/healthz"
+    validate_certs: false
+  when: inventory_hostname == groups['kube-master']|first
+  register: _result
+  retries: 60
+  delay: 5
+  until: _result.status == 200
+
 - name: kubeadm | Upgrade first master
   command: >-
     timeout -k 600s 600s

upgrade-cluster.yml

@@ -30,16 +30,24 @@
     - { role: kubespray-defaults}
     - { role: bootstrap-os, tags: bootstrap-os}
 
-- hosts: k8s-cluster:etcd:calico-rr
+- name: Prepare nodes for upgrade
+  hosts: k8s-cluster:etcd:calico-rr
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  serial: "{{ serial | default('20%') }}"
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/preinstall, tags: preinstall }
-    - { role: container-engine, tags: "container-engine", when: deploy_container_engine|default(true) }
     - { role: download, tags: download, when: "not skip_downloads" }
   environment: "{{ proxy_env }}"
 
+- name: Upgrade container engine on non-cluster nodes
+  hosts: etcd:calico-rr:!k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  serial: "{{ serial | default('20%') }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: container-engine, tags: "container-engine", when: deploy_container_engine|default(true) }
+  environment: "{{ proxy_env }}"
+
 - hosts: etcd
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
@@ -69,6 +77,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
+    - { role: container-engine, tags: "container-engine", when: deploy_container_engine|default(true) }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/master, tags: master, upgrade_cluster_setup: true }
     - { role: kubernetes/client, tags: client }
@@ -94,6 +103,7 @@
   roles:
     - { role: kubespray-defaults}
     - { role: upgrade/pre-upgrade, tags: pre-upgrade }
+    - { role: container-engine, tags: "container-engine", when: deploy_container_engine|default(true) }
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }