Fix vault setup partially (#1531)

This does not address per-node certs and scheduler/proxy/controller-manager
component certs which are now required. This should be handled in a
follow-up patch.

contrib/inventory_builder/inventory.py

@@ -41,7 +41,7 @@ import re
 import sys
 
 ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children',
-         'calico-rr']
+         'calico-rr', 'vault']
 PROTECTED_NAMES = ROLES
 AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load']
 _boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
@@ -250,6 +250,7 @@ class KubesprayInventory(object):
     def set_etcd(self, hosts):
         for host in hosts:
             self.add_host_to_group('etcd', host)
+            self.add_host_to_group('vault', host)
 
     def load_file(self, files=None):
         '''Directly loads JSON, or YAML file to inventory.'''

roles/etcd/tasks/gen_certs_vault.yml

@@ -11,12 +11,12 @@
 - name: gen_certs_vault | Read in the local credentials
   command: cat /etc/vault/roles/etcd/userpass
   register: etcd_vault_creds_cat
-  when: inventory_hostname == groups.etcd|first
+  delegate_to: "{{ groups['vault'][0] }}"
 
 - name: gen_certs_vault | Set facts for read Vault Creds
   set_fact:
-    etcd_vault_creds: "{{ hostvars[groups.etcd|first]['etcd_vault_creds_cat']['stdout']|from_json }}"
-  when: inventory_hostname == groups.etcd|first
+    etcd_vault_creds: "{{ etcd_vault_creds_cat.stdout|from_json }}"
+  delegate_to: "{{ groups['vault'][0] }}"
 
 - name: gen_certs_vault | Log into Vault and obtain an token
   uri:
@@ -29,12 +29,12 @@
     body:
       password: "{{ etcd_vault_creds.password }}"
   register: etcd_vault_login_result
-  when: inventory_hostname == groups.etcd|first
+  delegate_to: "{{ groups['vault'][0] }}"
 
 - name: gen_certs_vault | Set fact for vault_client_token
   set_fact:
     vault_client_token:  "{{ etcd_vault_login_result.get('json', {}).get('auth', {}).get('client_token') }}"
-  delegate_to: "{{ groups['etcd'][0] }}"
+  run_once: true
 
 - name: gen_certs_vault | Set fact for Vault API token
   set_fact:
@@ -42,6 +42,7 @@
         Accept: application/json
         Content-Type: application/json
         X-Vault-Token: "{{ vault_client_token }}"
+  run_once: true
   when: vault_client_token != ""
 
 # Issue master certs to Etcd nodes

roles/vault/tasks/cluster/unseal.yml

 ---
 
+- name: cluster/unseal | Current sealed state
+  debug: " Sealed? {{vault_is_sealed}}"
+
 - name: cluster/unseal | Unseal Vault
   uri:
     url: "https://localhost:{{ vault_port }}/v1/sys/unseal"

roles/vault/tasks/shared/check_etcd.yml

 ---
 
-- name: check_etcd | Check if etcd is up an reachable
+- name: check_etcd | Check if etcd is up and reachable
   uri:
     url: "{{ vault_etcd_url }}/health"
     validate_certs: no
+  until: vault_etcd_health_check.status == 200 or vault_etcd_health_check.status == 401
+  retries: 10
+  delay: 2
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
   failed_when: false
   register: vault_etcd_health_check
 

roles/vault/tasks/shared/check_vault.yml

@@ -12,7 +12,7 @@
   uri:
     url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
     headers: "{{ vault_client_headers }}"
-    status_code: 200,429,500,501
+    status_code: 200,429,500,501,503
     validate_certs: no
   ignore_errors: true
   register: vault_local_service_health

roles/vault/tasks/shared/find_leader.yml

@@ -5,7 +5,7 @@
     url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://localhost:{{ vault_port }}/v1/sys/health"
     headers: "{{ hostvars[groups.vault|first]['vault_headers'] }}"
     method: HEAD
-    status_code: 200,429
+    status_code: 200,429,503
   register: vault_leader_check
   until: "vault_leader_check|succeeded"
   retries: 10
@@ -14,7 +14,8 @@
   set_fact:
     vault_leader_url: "{{ vault_config.listener.tcp.tls_disable|d()|ternary('http', 'https') }}://{{ item }}:{{ vault_port }}"
   with_items: "{{ groups.vault }}"
-  when: "hostvars[item]['vault_leader_check'].get('status') == 200"
-  run_once: true
+  when: "hostvars[item]['vault_leader_check'].get('status') in [200,503]"
+  #run_once: true
 
-- debug: var=vault_leader_url verbosity=2
+- name: find_leader| show vault_leader_url
+  debug: var=vault_leader_url verbosity=2

roles/vault/tasks/shared/issue_cert.yml

@@ -18,6 +18,11 @@
 #   issue_cert_role:        The Vault role to issue the cert with
 #   issue_cert_url:         Url to reach Vault, including protocol and port
 
+- name: issue_cert | debug who issues certs
+  debug:
+    msg: "{{ issue_cert_hosts }} issues certs"
+
+
 - name: issue_cert | Ensure target directory exists
   file:
     path: "{{ issue_cert_path | dirname }}" 
@@ -38,11 +43,16 @@
       format: "{{ issue_cert_format | d('pem') }}"
       ip_sans: "{{ issue_cert_ip_sans | default([]) | join(',') }}"
   register: issue_cert_result
-  when: inventory_hostname == issue_cert_hosts|first
+  delegate_to: "{{ issue_cert_hosts|first }}"
+
+- name: issue_cert | results
+  debug:
+    msg: "{{ issue_cert_result }}"
+
 
 - name: "issue_cert | Copy {{ issue_cert_path }} cert to all hosts"
   copy:
-    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['certificate'] }}"
+    content: "{{ issue_cert_result['json']['data']['certificate'] }}"
     dest: "{{ issue_cert_path }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0644') }}"
@@ -50,7 +60,7 @@
 
 - name: "issue_cert | Copy key for {{ issue_cert_path }} to all hosts"
   copy:
-    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['private_key'] }}"
+    content: "{{ issue_cert_result['json']['data']['private_key'] }}"
     dest: "{{ issue_cert_path.rsplit('.', 1)|first }}-key.{{ issue_cert_path.rsplit('.', 1)|last }}"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0640') }}"
@@ -58,7 +68,7 @@
 
 - name: issue_cert | Copy issuing CA cert
   copy:
-    content: "{{ hostvars[issue_cert_hosts|first]['issue_cert_result']['json']['data']['issuing_ca'] }}"
+    content: "{{ issue_cert_result['json']['data']['issuing_ca'] }}"
     dest: "{{ issue_cert_path | dirname }}/ca.pem"
     group: "{{ issue_cert_file_group | d('root' )}}"
     mode: "{{ issue_cert_file_mode | d('0644') }}"