From e9f795c5cee624551ceef019f2466e29845bf2c4 Mon Sep 17 00:00:00 2001
From: Chad Swenson <chadswen@gmail.com>
Date: Wed, 8 Nov 2017 01:40:33 -0600
Subject: [PATCH] Master component and kubelet container upgrade fixes
* Fixes an issue where apiserver and friends (controller manager, scheduler) were prevented from restarting after manifests/secrets are changed. This occurred when a replaced kubelet doesn't reconcile new master manifests, which caused old master component versions to linger during deployment. In my case this was causing upgrades from k8s 1.6/1.7 -> k8s 1.8 to fail
* Improves transitions from kubelet container to host kubelet by preventing issues where kubelet container reappeared during the deployment
---
roles/kubernetes/master/handlers/main.yml | 27 +++++++++++++++++++
roles/kubernetes/master/tasks/pre-upgrade.yml | 2 +-
.../master/tasks/static-pod-setup.yml | 6 ++---
roles/kubernetes/node/tasks/pre_upgrade.yml | 19 ++++++++++---
4 files changed, 47 insertions(+), 7 deletions(-)
diff --git a/roles/kubernetes/master/handlers/main.yml b/roles/kubernetes/master/handlers/main.yml
index 1c6dc956c..dd3b03264 100644
--- a/roles/kubernetes/master/handlers/main.yml
+++ b/roles/kubernetes/master/handlers/main.yml
@@ -13,6 +13,24 @@
- Master | wait for kube-scheduler
- Master | wait for kube-controller-manager
+- name: Master | Restart apiserver
+ command: /bin/true
+ notify:
+ - Master | Remove apiserver container
+ - Master | wait for the apiserver to be running
+
+- name: Master | Restart kube-scheduler
+ command: /bin/true
+ notify:
+ - Master | Remove scheduler container
+ - Master | wait for kube-scheduler
+
+- name: Master | Restart kube-controller-manager
+ command: /bin/true
+ notify:
+ - Master | Remove controller manager container
+ - Master | wait for kube-controller-manager
+
- name: Master | reload systemd
command: systemctl daemon-reload
@@ -21,6 +39,15 @@
name: kubelet
state: restarted
+- name: Master | Remove apiserver container
+ shell: "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f"
+
+- name: Master | Remove scheduler container
+ shell: "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f"
+
+- name: Master | Remove controller manager container
+ shell: "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f"
+
- name: Master | wait for kube-scheduler
uri:
url: http://localhost:10251/healthz
diff --git a/roles/kubernetes/master/tasks/pre-upgrade.yml b/roles/kubernetes/master/tasks/pre-upgrade.yml
index b546f0ca9..81da33433 100644
--- a/roles/kubernetes/master/tasks/pre-upgrade.yml
+++ b/roles/kubernetes/master/tasks/pre-upgrade.yml
@@ -24,7 +24,7 @@
when: (secret_changed|default(false) or etcd_secret_changed|default(false))
- name: "Pre-upgrade | Delete master containers forcefully"
- shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f"
+ shell: "docker ps -af name=k8s_{{item}}* -q | xargs --no-run-if-empty docker rm -f"
with_items:
- ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
when: kube_apiserver_manifest_replaced.changed
diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml
index 79f95d860..8110cfe8c 100644
--- a/roles/kubernetes/master/tasks/static-pod-setup.yml
+++ b/roles/kubernetes/master/tasks/static-pod-setup.yml
@@ -3,7 +3,7 @@
template:
src: manifests/kube-apiserver.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
- notify: Master | wait for the apiserver to be running
+ notify: Master | Restart apiserver
tags:
- kube-apiserver
@@ -20,7 +20,7 @@
template:
src: manifests/kube-scheduler.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
- notify: Master | wait for kube-scheduler
+ notify: Master | Restart kube-scheduler
tags:
- kube-scheduler
@@ -35,6 +35,6 @@
template:
src: manifests/kube-controller-manager.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
- notify: Master | wait for kube-controller-manager
+ notify: Master | Restart kube-controller-manager
tags:
- kube-controller-manager
diff --git a/roles/kubernetes/node/tasks/pre_upgrade.yml b/roles/kubernetes/node/tasks/pre_upgrade.yml
index 2701808be..8919bf0be 100644
--- a/roles/kubernetes/node/tasks/pre_upgrade.yml
+++ b/roles/kubernetes/node/tasks/pre_upgrade.yml
@@ -1,12 +1,25 @@
---
+- name: "Pre-upgrade | check if kubelet container exists"
+ shell: docker ps -af name=kubelet | grep kubelet
+ failed_when: false
+ changed_when: false
+ register: kubelet_container_check
+
- name: "Pre-upgrade | copy /var/lib/cni from kubelet"
command: docker cp kubelet:/var/lib/cni /var/lib/cni
args:
creates: "/var/lib/cni"
failed_when: false
+ when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
+ service:
+ name: kubelet
+ state: stopped
+ when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
-- name: "Pre-upgrade | ensure kubelet container is stopped if using host deployment"
- command: docker stop kubelet
+- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
+ command: docker rm -fv kubelet
failed_when: false
changed_when: false
- when: kubelet_deployment_type == 'host'
+ when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
\ No newline at end of file
--
GitLab