From e8a1c7a53f9e81b425f5bf12b549ddb199562d1b Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Tue, 2 Aug 2016 13:42:27 +0300
Subject: [PATCH] Move docker systemd unit creation to docker role

Creating the unit using default settings early on
and then changing it during network_plugin section
leads to too many docker restarts and duplicated code.

Reversed Wants= dependence on docker.service so it does not
restart docker when reloading systemd

Consolidated all docker restart handlers.
---
 roles/docker/handlers/main.yml                | 12 +++++++
 roles/docker/tasks/main.yml                   |  8 +++++
 .../templates/systemd-docker.service.j2}      |  3 +-
 roles/etcd/templates/etcd-docker.service.j2   |  6 ++--
 .../templates/etcd-proxy-docker.service.j2    |  6 ++--
 .../node/templates/kubelet.service.j2         |  8 ++---
 roles/network_plugin/calico/tasks/main.yml    |  7 ----
 .../calico/templates/calico-node.service.j2   |  2 +-
 .../network_plugin/flannel/handlers/main.yml  | 16 ----------
 roles/network_plugin/flannel/tasks/main.yml   |  7 ----
 .../flannel/templates/systemd-docker.service  | 32 -------------------
 roles/network_plugin/meta/main.yml            |  1 +
 roles/network_plugin/weave/handlers/main.yml  | 11 -------
 roles/network_plugin/weave/tasks/main.yml     |  9 +-----
 .../weave/templates/systemd-docker.service    | 32 -------------------
 .../weave/templates/weave.service.j2          |  2 +-
 .../weave/templates/weaveexpose.service.j2    |  2 +-
 .../weave/templates/weaveproxy.service.j2     |  2 +-
 18 files changed, 38 insertions(+), 128 deletions(-)
 rename roles/{network_plugin/calico/templates/systemd-docker.service => docker/templates/systemd-docker.service.j2} (96%)
 delete mode 100644 roles/network_plugin/flannel/templates/systemd-docker.service
 delete mode 100644 roles/network_plugin/weave/templates/systemd-docker.service

diff --git a/roles/docker/handlers/main.yml b/roles/docker/handlers/main.yml
index 79db42f92..6f54f33d5 100644
--- a/roles/docker/handlers/main.yml
+++ b/roles/docker/handlers/main.yml
@@ -4,6 +4,8 @@
   notify:
     - Docker | reload systemd
     - Docker | reload docker
+    - Docker | pause while Docker restarts
+    - Docker | wait for docker
 
 - name : Docker | reload systemd
   shell: systemctl daemon-reload
@@ -13,3 +15,13 @@
   service:
     name: docker
     state: restarted
+
+- name: Docker | pause while Docker restarts
+  pause: seconds=10 prompt="Waiting for docker restart"
+
+- name: Docker | wait for docker
+  command: /usr/bin/docker images
+  register: docker_ready
+  retries: 10
+  delay: 5
+  until: docker_ready.rc == 0
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 03db20c75..6d71c4980 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -59,6 +59,14 @@
   when: ansible_service_mgr == "systemd" and
         (http_proxy is defined or https_proxy is defined or no_proxy is defined)
 
+- name: Write docker.service systemd file
+  template:
+    src: systemd-docker.service.j2
+    dest: /etc/systemd/system/docker.service
+  register: docker_service_file
+  notify: restart docker
+  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
+
 - meta: flush_handlers
 
 - name: ensure docker service is started and enabled
diff --git a/roles/network_plugin/calico/templates/systemd-docker.service b/roles/docker/templates/systemd-docker.service.j2
similarity index 96%
rename from roles/network_plugin/calico/templates/systemd-docker.service
rename to roles/docker/templates/systemd-docker.service.j2
index d20a2fbe0..b19b1caaf 100644
--- a/roles/network_plugin/calico/templates/systemd-docker.service
+++ b/roles/docker/templates/systemd-docker.service.j2
@@ -29,11 +29,12 @@ ExecStart=/usr/bin/docker daemon \
           $DOCKER_NETWORK_OPTIONS \
           $INSECURE_REGISTRY \
           $DOCKER_OPTS
+TasksMax=infinity
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitCORE=infinity
-MountFlags=slave
 TimeoutStartSec=1min
+Restart=on-abnormal
 
 [Install]
 WantedBy=multi-user.target
diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2
index 4b6cec5c9..a37759fec 100644
--- a/roles/etcd/templates/etcd-docker.service.j2
+++ b/roles/etcd/templates/etcd-docker.service.j2
@@ -1,7 +1,7 @@
 [Unit]
 Description=etcd docker wrapper
-Wants=docker.service docker.socket
-After=docker.service docker.socket
+Wants=docker.socket
+After=docker.service
 
 [Service]
 User=root
@@ -18,7 +18,7 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
 {% if etcd_after_v3 %}
 {{ etcd_container_bin_dir }}etcd
 {% endif %}
-ExecStopPost=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }}
+ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_member_name | default("etcd-proxy") }}
 ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_member_name | default("etcd-proxy") }}
 ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_member_name | default("etcd-proxy") }}
 Restart=always
diff --git a/roles/etcd/templates/etcd-proxy-docker.service.j2 b/roles/etcd/templates/etcd-proxy-docker.service.j2
index 939e6fd35..bf70f0e7f 100644
--- a/roles/etcd/templates/etcd-proxy-docker.service.j2
+++ b/roles/etcd/templates/etcd-proxy-docker.service.j2
@@ -1,7 +1,7 @@
 [Unit]
 Description=etcd-proxy docker wrapper
-Wants=docker.service docker.socket
-After=docker.service docker.socket
+Wants=docker.socket
+After=docker.service
 
 [Service]
 User=root
@@ -18,7 +18,7 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
 {% if etcd_after_v3 %}
 {{ etcd_container_bin_dir }}etcd
 {% endif %}
-ExecStopPost=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }}
+ExecStartPre=-{{ docker_bin_dir | default("/usr/bin") }}/docker rm -f {{ etcd_proxy_member_name | default("etcd-proxy") }}
 ExecReload={{ docker_bin_dir | default("/usr/bin") }}/docker restart {{ etcd_proxy_member_name | default("etcd-proxy") }}
 ExecStop={{ docker_bin_dir | default("/usr/bin") }}/docker stop {{ etcd_proxy_member_name | default("etcd-proxy") }}
 Restart=always
diff --git a/roles/kubernetes/node/templates/kubelet.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2
index e92f71d51..ad62d8562 100644
--- a/roles/kubernetes/node/templates/kubelet.service.j2
+++ b/roles/kubernetes/node/templates/kubelet.service.j2
@@ -3,10 +3,10 @@ Description=Kubernetes Kubelet Server
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
 After=docker.service docker.socket calico-node.service
-Wants=docker.service docker.socket calico-node.service
+Wants=docker.socket calico-node.service
 {% else %}
-After=docker.service docker.socket
-Wants=docker.service docker.socket
+After=docker.service
+Wants=docker.socket
 {% endif %}
 
 [Service]
@@ -24,7 +24,7 @@ ExecStart={{ bin_dir }}/kubelet \
 		$KUBELET_REGISTER_NODE \
 		$KUBELET_NETWORK_PLUGIN \
 		$KUBELET_CLOUDPROVIDER
-ExecStopPost=-/usr/bin/docker rm -f kubelet
+ExecStartPre=-/usr/bin/docker rm -f kubelet
 ExecReload=/usr/bin/docker restart kubelet
 Restart=always
 RestartSec=10s
diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index a4c32c1c0..510b8af82 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -10,13 +10,6 @@
     - restart docker
   when: ansible_os_family != "CoreOS"
 
-- name: Calico | Write docker.service systemd file
-  template:
-    src: systemd-docker.service
-    dest: /lib/systemd/system/docker.service
-  notify: restart docker
-  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
-
 - meta: flush_handlers
 
 - name: Calico | Install calicoctl container script
diff --git a/roles/network_plugin/calico/templates/calico-node.service.j2 b/roles/network_plugin/calico/templates/calico-node.service.j2
index 152ecce64..4c709705d 100644
--- a/roles/network_plugin/calico/templates/calico-node.service.j2
+++ b/roles/network_plugin/calico/templates/calico-node.service.j2
@@ -2,7 +2,7 @@
 Description=Calico per-node agent
 Documentation=https://github.com/projectcalico/calico-docker
 After=docker.service docker.socket etcd-proxy.service
-Wants=docker.service docker.socket etcd-proxy.service
+Wants=docker.socket etcd-proxy.service
 
 [Service]
 User=root
diff --git a/roles/network_plugin/flannel/handlers/main.yml b/roles/network_plugin/flannel/handlers/main.yml
index 1982765d4..a503569f6 100644
--- a/roles/network_plugin/flannel/handlers/main.yml
+++ b/roles/network_plugin/flannel/handlers/main.yml
@@ -4,22 +4,6 @@
   ignore_errors: yes
   notify: restart docker
 
-- name: restart docker
-  command: /bin/true
-  notify:
-    - Flannel | reload systemd
-    - Flannel | reload docker
-    - Flannel | reload kubelet
-
-- name : Flannel | reload systemd
-  shell: systemctl daemon-reload
-  when: ansible_service_mgr == "systemd"
-
-- name: Flannel | reload docker
-  service:
-    name: docker
-    state: restarted
-
 - name: Flannel | reload kubelet
   service:
     name: kubelet
diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml
index 55c47d211..9588559f6 100644
--- a/roles/network_plugin/flannel/tasks/main.yml
+++ b/roles/network_plugin/flannel/tasks/main.yml
@@ -50,11 +50,4 @@
     state: link
   when: ansible_os_family == "CoreOS"
 
-- name: Flannel | Write docker.service systemd file
-  template:
-    src: systemd-docker.service
-    dest: /lib/systemd/system/docker.service
-  notify: restart docker
-  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
-
 - meta: flush_handlers
diff --git a/roles/network_plugin/flannel/templates/systemd-docker.service b/roles/network_plugin/flannel/templates/systemd-docker.service
deleted file mode 100644
index 21790dd6f..000000000
--- a/roles/network_plugin/flannel/templates/systemd-docker.service
+++ /dev/null
@@ -1,32 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=http://docs.docker.com
-{% if ansible_os_family == "RedHat" %}
-After=network.target docker-storage-setup.service
-Wants=docker-storage-setup.service
-{% elif ansible_os_family == "Debian" %}
-After=network.target docker.socket
-Wants=docker.socket
-{% endif %}
-
-[Service]
-Type=notify
-EnvironmentFile=-/etc/default/docker
-Environment=GOTRACEBACK=crash
-ExecReload=/bin/kill -s HUP $MAINPID
-Delegate=yes
-KillMode=process
-ExecStart=/usr/bin/docker daemon \
-          $OPTIONS \
-          $DOCKER_STORAGE_OPTIONS \
-          $DOCKER_NETWORK_OPTIONS \
-          $INSECURE_REGISTRY \
-          $DOCKER_OPTS
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-MountFlags=slave
-TimeoutStartSec=1min
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/network_plugin/meta/main.yml b/roles/network_plugin/meta/main.yml
index 736262ab0..0dd36511b 100644
--- a/roles/network_plugin/meta/main.yml
+++ b/roles/network_plugin/meta/main.yml
@@ -6,3 +6,4 @@ dependencies:
    when: kube_network_plugin == 'flannel'
  - role: network_plugin/weave
    when: kube_network_plugin == 'weave'
+ - role: docker
diff --git a/roles/network_plugin/weave/handlers/main.yml b/roles/network_plugin/weave/handlers/main.yml
index f983ee579..e821e989b 100644
--- a/roles/network_plugin/weave/handlers/main.yml
+++ b/roles/network_plugin/weave/handlers/main.yml
@@ -1,10 +1,4 @@
 ---
-- name: Weave | restart docker
-  command: /bin/true
-  notify:
-    - Weave | reload systemd
-    - Weave | reload docker
-
 - name: restart weave
   command: /bin/true
   notify:
@@ -27,11 +21,6 @@
     - Weave | reload systemd
     - reload weaveexpose
 
-- name: Weave | reload docker
-  service:
-    name: docker
-    state: restarted
-
 - name: reload weave
   service:
     name: weave
diff --git a/roles/network_plugin/weave/tasks/main.yml b/roles/network_plugin/weave/tasks/main.yml
index 5703dde97..eb56d8c9b 100644
--- a/roles/network_plugin/weave/tasks/main.yml
+++ b/roles/network_plugin/weave/tasks/main.yml
@@ -7,14 +7,7 @@
     group: root
     mode: 0644
   notify:
-    - Weave | restart docker
-
-- name: Write docker.service systemd file
-  template:
-    src: systemd-docker.service
-    dest: /lib/systemd/system/docker.service
-  notify: Weave | restart docker
-  when: ansible_service_mgr == "systemd" and ansible_os_family != "CoreOS"
+    - restart docker
 
 - name: Weave | Install weave
   command: rsync -piu "{{ local_release_dir }}/weave/bin/weave" "{{ bin_dir }}/weave"
diff --git a/roles/network_plugin/weave/templates/systemd-docker.service b/roles/network_plugin/weave/templates/systemd-docker.service
deleted file mode 100644
index 96dd6cd05..000000000
--- a/roles/network_plugin/weave/templates/systemd-docker.service
+++ /dev/null
@@ -1,32 +0,0 @@
-[Unit]
-Description=Docker Application Container Engine
-Documentation=http://docs.docker.com
-{% if ansible_os_family == "RedHat" %}
-After=network.target
-Wants=docker-storage-setup.service
-{% elif ansible_os_family == "Debian" %}
-After=network.target docker.socket
-Wants=docker.socket
-{% endif %}
-
-[Service]
-Type=notify
-EnvironmentFile=-/etc/default/docker
-Environment=GOTRACEBACK=crash
-ExecReload=/bin/kill -s HUP $MAINPID
-Delegate=yes
-KillMode=process
-ExecStart=/usr/bin/docker daemon \
-          $OPTIONS \
-          $DOCKER_STORAGE_OPTIONS \
-          $DOCKER_NETWORK_OPTIONS \
-          $INSECURE_REGISTRY \
-          $DOCKER_OPTS
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitCORE=infinity
-MountFlags=slave
-TimeoutStartSec=1min
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/network_plugin/weave/templates/weave.service.j2 b/roles/network_plugin/weave/templates/weave.service.j2
index 46d9434fe..e901c34e7 100644
--- a/roles/network_plugin/weave/templates/weave.service.j2
+++ b/roles/network_plugin/weave/templates/weave.service.j2
@@ -1,7 +1,7 @@
 [Unit]
 Description=Weave Network
 Documentation=http://docs.weave.works/weave/latest_release/
-Wants=docker.service docker.socket
+Wants=docker.socket
 After=docker.service docker.socket
 
 [Service]
diff --git a/roles/network_plugin/weave/templates/weaveexpose.service.j2 b/roles/network_plugin/weave/templates/weaveexpose.service.j2
index 912ed1fee..f9931696e 100644
--- a/roles/network_plugin/weave/templates/weaveexpose.service.j2
+++ b/roles/network_plugin/weave/templates/weaveexpose.service.j2
@@ -1,6 +1,6 @@
 [Unit]
 Documentation=http://docs.weave.works/
-Wants=docker.service docker.socket weave.service
+Wants=docker.socket weave.service
 After=docker.service docker.socket weave.service
 
 [Service]
diff --git a/roles/network_plugin/weave/templates/weaveproxy.service.j2 b/roles/network_plugin/weave/templates/weaveproxy.service.j2
index f37120f1c..29197296f 100644
--- a/roles/network_plugin/weave/templates/weaveproxy.service.j2
+++ b/roles/network_plugin/weave/templates/weaveproxy.service.j2
@@ -1,7 +1,7 @@
 [Unit]
 Description=Weave proxy for Docker API
 Documentation=http://docs.weave.works/
-Wants=docker.service docker.socket
+Wants=docker.socket
 After=docker.service docker.socket
 
 [Service]
-- 
GitLab