Scale-up functionality for etcd cluster

* Set ETCD_INITIAL_CLUSTER_STATE from `new` to `existing`,
because parameter `new` makes sense only on cluster assembly
stage.
* If cluster exists and current node is not a part
of the cluster, add it with command `etcdctl add member name url`.

Closes kubespray/kargo/#270

roles/etcd/tasks/configure.yml

 ---
-- name: Configure |  Copy etcd.service systemd file
+- name: Configure | Check if member is in cluster
+  shell: "etcdctl --peers={{ etcd_access_addresses }} member list | grep -q {{ etcd_access_address }}"
+  register: etcd_member_in_cluster
+  ignore_errors: true
+  changed_when: false
+  when: is_etcd_master
+
+- name: Configure | Add member to the cluster if it is not there
+  when: is_etcd_master and etcd_member_in_cluster.rc != 0 and etcd_cluster_is_healthy.rc == 0
+  shell: "etcdctl --peers={{ etcd_access_addresses }} member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+
+- name: Configure | Copy etcd.service systemd file
   template:
     src: etcd.service.j2
     dest: /etc/systemd/system/etcd.service
@@ -7,7 +18,7 @@
   when: ansible_service_mgr == "systemd"
   notify: restart etcd
 
-- name: Configure |  Write etcd  initd script
+- name: Configure | Write etcd initd script
   template:
     src: deb-etcd.initd.j2
     dest: /etc/init.d/etcd
@@ -15,9 +26,3 @@
     mode: 0755
   when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "Debian"
   notify: restart etcd
-
-- name: Configure |  Create etcd config file
-  template:
-    src: etcd.j2
-    dest: /etc/etcd.env
-  notify: restart etcd

roles/etcd/tasks/main.yml

 ---
+- include: set_facts.yml
 - include: install.yml
+- include: set_cluster_health.yml
 - include: configure.yml
+- include: refresh_config.yml
 
 - name: Restart etcd if binary changed
   command: /bin/true
   notify: restart etcd
   when: etcd_copy.stdout_lines
 
-# reload systemd before starting service
+# Reload systemd before starting service
 - meta: flush_handlers
 
-
 - name: Ensure etcd is running
   service:
     name: etcd
     state: started
     enabled: yes
+
+# After etcd cluster is assembled, make sure that
+# initial state of the cluster is in `existing`
+# state insted of `new`.
+- include: set_cluster_health.yml
+- include: refresh_config.yml

roles/etcd/tasks/refresh_config.yml

+---
+- name: Refresh config | Create etcd config file
+  template:
+    src: etcd.j2
+    dest: /etc/etcd.env
+  notify: restart etcd

roles/etcd/tasks/set_cluster_health.yml

+---
+- name: Configure | Check if cluster is healthy
+  shell: "etcdctl --peers={{ etcd_access_addresses }} cluster-health | grep -q 'cluster is healthy'"
+  register: etcd_cluster_is_healthy
+  ignore_errors: true
+  changed_when: false
+  when: is_etcd_master

roles/etcd/tasks/set_facts.yml

+---
+- set_fact: etcd_access_address="{{ access_ip | default(ip | default(ansible_default_ipv4['address'])) }}"
+- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
+- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
+- set_fact:
+    etcd_access_addresses: |-
+      {% for item in groups['etcd'] -%}
+        http://{{ hostvars[item].etcd_access_address }}:2379{% if not loop.last %},{% endif %}
+      {%- endfor %}
+- set_fact:
+    etcd_member_name: |-
+      {% for host in groups['etcd'] %}
+      {%   if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %}
+      {% endfor %}
+- set_fact:
+    is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"

roles/etcd/templates/etcd.j2

 ETCD_DATA_DIR="/var/lib/etcd"
 {% if inventory_hostname in groups['etcd'] %}
-{% set etcd = {} %}
-{%     for host in groups['etcd'] %}
-{%         if inventory_hostname == host %}
-{%             set _dummy = etcd.update({'name':"etcd"+loop.index|string}) %}
-{%         endif %}
-{%     endfor %}
 ETCD_ADVERTISE_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])) }}:2379"
 ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{{ hostvars[inventory_hostname]['access_ip'] | default(hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address']))  }}:2380"
-ETCD_INITIAL_CLUSTER_STATE="new"
+ETCD_INITIAL_CLUSTER_STATE="{% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}"
 ETCD_INITIAL_CLUSTER_TOKEN="k8s_etcd"
 ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])  }}:2380"
-ETCD_NAME="{{ etcd.name }}"
+ETCD_NAME="{{ etcd_member_name }}"
 {% endif %}
-ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}=http://{{ hostvars[host]['access_ip'] | default(hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}:2380{% if not loop.last %},{% endif %}{% endfor %}"
+
+ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}etcd{{ loop.index|string }}={{ hostvars[host]['etcd_peer_url'] }}{% if not loop.last %},{% endif %}{% endfor %}"
 ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( hostvars[inventory_hostname]['ansible_default_ipv4']['address'])  }}:2379,http://127.0.0.1:2379"