From e78562830f353be20a8627ce46ffcb80850f86be Mon Sep 17 00:00:00 2001 From: Chad Swenson <chadswen@gmail.com> Date: Tue, 12 Dec 2017 12:06:41 -0600 Subject: [PATCH] Retry kube container removal during upgrade As we have seen with other containers, sometimes container removal fails on the first attempt due to some Docker bugs. Retrying typically corrects the issue. --- roles/kubernetes/master/handlers/main.yml | 12 ++++++++++++ roles/kubernetes/node/tasks/pre_upgrade.yml | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml index dd3b03264..070250cb0 100644 --- a/roles/kubernetes/master/handlers/main.yml +++ b/roles/kubernetes/master/handlers/main.yml @@ -41,12 +41,24 @@ - name: Master | Remove apiserver container shell: "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f" + register: remove_apiserver_container + retries: 4 + until: remove_apiserver_container.rc == 0 + delay: 5 - name: Master | Remove scheduler container shell: "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f" + register: remove_scheduler_container + retries: 4 + until: remove_scheduler_container.rc == 0 + delay: 5 - name: Master | Remove controller manager container shell: "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f" + register: remove_cm_container + retries: 4 + until: remove_cm_container.rc == 0 + delay: 5 - name: Master | wait for kube-scheduler uri: diff --git a/roles/kubernetes/node/tasks/pre_upgrade.yml b/roles/kubernetes/node/tasks/pre_upgrade.yml index 8919bf0be..6d24b006b 100644 --- a/roles/kubernetes/node/tasks/pre_upgrade.yml +++ b/roles/kubernetes/node/tasks/pre_upgrade.yml @@ -22,4 +22,8 @@ command: docker rm -fv kubelet failed_when: false changed_when: false + register: remove_kubelet_container + retries: 4 + until: remove_kubelet_container.rc == 0 + delay: 5 when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0 \ No newline at end of file -- GitLab