From fa378f09c3027c43f21bad17463a720fb831c926 Mon Sep 17 00:00:00 2001
From: Bernard Landon <bernard.landon@gmail.com>
Date: Thu, 20 Aug 2020 13:25:36 +0200
Subject: [PATCH] Edited pre-upgrade task to uncordon a node failing to drain
(#6546)
---
roles/upgrade/pre-upgrade/tasks/main.yml | 68 +++++++++++++-----------
1 file changed, 37 insertions(+), 31 deletions(-)
diff --git a/roles/upgrade/pre-upgrade/tasks/main.yml b/roles/upgrade/pre-upgrade/tasks/main.yml
index e67c4a52e..5be402f95 100644
--- a/roles/upgrade/pre-upgrade/tasks/main.yml
+++ b/roles/upgrade/pre-upgrade/tasks/main.yml
@@ -30,40 +30,46 @@
false
{%- endif %}
-- name: Cordon node
- command: "{{ bin_dir }}/kubectl cordon {{ kube_override_hostname|default(inventory_hostname) }}"
- delegate_to: "{{ groups['kube-master'][0] }}"
- when: needs_cordoning
+- name: Node draining
+ block:
+ - name: Cordon node
+ command: "{{ bin_dir }}/kubectl cordon {{ kube_override_hostname|default(inventory_hostname) }}"
+ delegate_to: "{{ groups['kube-master'][0] }}"
-- name: Check kubectl version
- command: "{{ bin_dir }}/kubectl version --client --short"
- register: kubectl_version
- delegate_to: "{{ groups['kube-master'][0] }}"
- run_once: yes
- changed_when: false
- when:
- - drain_nodes
- - needs_cordoning
- - drain_pod_selector
+ - name: Check kubectl version
+ command: "{{ bin_dir }}/kubectl version --client --short"
+ register: kubectl_version
+ delegate_to: "{{ groups['kube-master'][0] }}"
+ run_once: yes
+ changed_when: false
+ when:
+ - drain_nodes
+ - drain_pod_selector
-- name: Ensure minimum version for drain label selector if necessary
- assert:
- that: "kubectl_version.stdout.split(' ')[-1] is version('v1.10.0', '>=')"
- when:
- - drain_nodes
- - needs_cordoning
- - drain_pod_selector
+ - name: Ensure minimum version for drain label selector if necessary
+ assert:
+ that: "kubectl_version.stdout.split(' ')[-1] is version('v1.10.0', '>=')"
+ when:
+ - drain_nodes
+ - drain_pod_selector
-- name: Drain node
- command: >-
- {{ bin_dir }}/kubectl drain
- --force
- --ignore-daemonsets
- --grace-period {{ drain_grace_period }}
- --timeout {{ drain_timeout }}
- --delete-local-data {{ kube_override_hostname|default(inventory_hostname) }}
- {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %}
+ - name: Drain node
+ command: >-
+ {{ bin_dir }}/kubectl drain
+ --force
+ --ignore-daemonsets
+ --grace-period {{ drain_grace_period }}
+ --timeout {{ drain_timeout }}
+ --delete-local-data {{ kube_override_hostname|default(inventory_hostname) }}
+ {% if drain_pod_selector %}--pod-selector '{{ drain_pod_selector }}'{% endif %}
+ when:
+ - drain_nodes
+ rescue:
+ - name: Set node back to schedulable
+ command: "{{ bin_dir }}/kubectl --kubeconfig /etc/kubernetes/admin.conf uncordon {{ inventory_hostname }}"
+ - name: Fail after rescue
+ fail:
+ msg: "Failed to drain node {{ inventory_hostname }}"
delegate_to: "{{ groups['kube-master'][0] }}"
when:
- - drain_nodes
- needs_cordoning
--
GitLab