Add retries to drain during upgrade. Allow leaving nodes cordoned after drain...

Add retries to drain during upgrade. Allow leaving nodes cordoned after drain failure. Allow continuing upgrade if drain fails. (#7206)

roles/upgrade/pre-upgrade/defaults/main.yml

@@ -3,6 +3,11 @@ drain_grace_period: 300
 drain_timeout: 360s
 drain_pod_selector: ""
 drain_nodes: true
+drain_retries: 3
+drain_retry_delay_seconds: 10
+
+upgrade_node_uncordon_after_drain_failure: true
+upgrade_node_fail_if_drain_fails: true
 
 upgrade_node_confirm: false
 upgrade_node_pause_seconds: 0

roles/upgrade/pre-upgrade/tasks/main.yml

@@ -77,14 +77,19 @@
         --timeout {{ drain_timeout }}
         --delete-local-data {{ kube_override_hostname|default(inventory_hostname) }}
         {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %}
-      when:
-        - drain_nodes
+      when: drain_nodes
+      register: result
+      until: result.rc == 0
+      retries: "{{ drain_retries }}"
+      delay: "{{ drain_retry_delay_seconds }}"
   rescue:
     - name: Set node back to schedulable
       command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf uncordon {{ inventory_hostname }}"
+      when: upgrade_node_uncordon_after_drain_failure
     - name: Fail after rescue
       fail:
         msg: "Failed to drain node {{ inventory_hostname }}"
+      when: upgrade_node_fail_if_drain_fails
   delegate_to: "{{ groups['kube-master'][0] }}"
   when:
     - needs_cordoning