Master component and kubelet container upgrade fixes

* Fixes an issue where apiserver and friends (controller manager, scheduler) were prevented from restarting after manifests/secrets are changed. This occurred when a replaced kubelet doesn't reconcile new master manifests, which caused old master component versions to linger during deployment. In my case this was causing upgrades from k8s 1.6/1.7 -> k8s 1.8 to fail
* Improves transitions from kubelet container to host kubelet by preventing issues where kubelet container reappeared during the deployment

roles/kubernetes/master/handlers/main.yml

@@ -13,6 +13,24 @@
     - Master | wait for kube-scheduler
     - Master | wait for kube-controller-manager
 
+- name: Master | Restart apiserver
+  command: /bin/true
+  notify:
+    - Master | Remove apiserver container
+    - Master | wait for the apiserver to be running
+
+- name: Master | Restart kube-scheduler
+  command: /bin/true
+  notify:
+    - Master | Remove scheduler container
+    - Master | wait for kube-scheduler
+
+- name: Master | Restart kube-controller-manager
+  command: /bin/true
+  notify:
+    - Master | Remove controller manager container
+    - Master | wait for kube-controller-manager
+
 - name: Master | reload systemd
   command: systemctl daemon-reload
 
@@ -21,6 +39,15 @@
     name: kubelet
     state: restarted
 
+- name: Master | Remove apiserver container
+  shell: "docker ps -af name=k8s_kube-apiserver* -q | xargs --no-run-if-empty docker rm -f"
+
+- name: Master | Remove scheduler container
+  shell: "docker ps -af name=k8s_kube-scheduler* -q | xargs --no-run-if-empty docker rm -f"
+
+- name: Master | Remove controller manager container
+  shell: "docker ps -af name=k8s_kube-controller-manager* -q | xargs --no-run-if-empty docker rm -f"
+
 - name: Master | wait for kube-scheduler
   uri:
     url: http://localhost:10251/healthz

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -24,7 +24,7 @@
   when: (secret_changed|default(false) or etcd_secret_changed|default(false))
 
 - name: "Pre-upgrade | Delete master containers forcefully"
-  shell: "docker ps -f name=k8s-{{item}}* -q | xargs --no-run-if-empty docker rm -f"
+  shell: "docker ps -af name=k8s_{{item}}* -q | xargs --no-run-if-empty docker rm -f"
   with_items:
     - ["kube-apiserver", "kube-controller-manager", "kube-scheduler"]
   when: kube_apiserver_manifest_replaced.changed

roles/kubernetes/master/tasks/static-pod-setup.yml

@@ -3,7 +3,7 @@
   template:
     src: manifests/kube-apiserver.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | wait for the apiserver to be running
+  notify: Master | Restart apiserver
   tags:
     - kube-apiserver
 
@@ -20,7 +20,7 @@
   template:
     src: manifests/kube-scheduler.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | wait for kube-scheduler
+  notify: Master | Restart kube-scheduler
   tags:
     - kube-scheduler
 
@@ -35,6 +35,6 @@
   template:
     src: manifests/kube-controller-manager.manifest.j2
     dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
+  notify: Master | Restart kube-controller-manager
   tags:
     - kube-controller-manager

roles/kubernetes/node/tasks/pre_upgrade.yml

 ---
+- name: "Pre-upgrade | check if kubelet container exists"
+  shell: docker ps -af name=kubelet | grep kubelet
+  failed_when: false
+  changed_when: false
+  register: kubelet_container_check
+
 - name: "Pre-upgrade | copy /var/lib/cni from kubelet"
   command: docker cp kubelet:/var/lib/cni /var/lib/cni
   args:
     creates: "/var/lib/cni"
   failed_when: false
+  when: kubelet_container_check.rc == 0
+
+- name: "Pre-upgrade | ensure kubelet container service is stopped if using host deployment"
+  service:
+    name: kubelet
+    state: stopped
+  when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
 
-- name: "Pre-upgrade | ensure kubelet container is stopped if using host deployment"
-  command: docker stop kubelet
+- name: "Pre-upgrade | ensure kubelet container is removed if using host deployment"
+  command: docker rm -fv kubelet
   failed_when: false
   changed_when: false
-  when: kubelet_deployment_type == 'host'
+  when: kubelet_deployment_type == 'host' and kubelet_container_check.rc == 0
\ No newline at end of file