Fixed deploy cluster with vault cert manager (#1548)

* Added custom ips to etcd vault distributed certificates

* Added custom ips to kube-master vault distributed certificates

* Added comment about issue_cert_copy_ca var in vault/issue_cert role file

* Generate kube-proxy, controller-manager and scheduler certificates by vault

* Revert "Disable vault from CI (#1546)"

This reverts commit 781f31d2.

* Fixed upgrade cluster with vault cert manager

* Remove vault dir in reset playbook

.gitlab-ci.yml

@@ -367,15 +367,14 @@ before_script:
   KUBELET_DEPLOYMENT: rkt
   STARTUP_SCRIPT: ""
 
-#Note(mattymo): Vault deployment is broken and needs work
-#.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
-## stage: deploy-gce-part1
-#  KUBE_NETWORK_PLUGIN: canal
-#  CERT_MGMT: vault
-#  CLOUD_IMAGE: ubuntu-1604-xenial
-#  CLOUD_REGION: us-central1-b
-#  CLUSTER_MODE: separate
-#  STARTUP_SCRIPT: ""
+.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  CERT_MGMT: vault
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_REGION: us-central1-b
+  CLUSTER_MODE: separate
+  STARTUP_SCRIPT: ""
 
 .ubuntu_flannel_rbac_variables: &ubuntu_flannel_rbac_variables
 # stage: deploy-gce-special
@@ -600,17 +599,16 @@ ubuntu-rkt-sep:
   except: ['triggers']
   only: ['master', /^pr-.*$/]
 
-#Note(mattymo): Vault deployment is broken (https://github.com/kubernetes-incubator/kubespray/issues/1545)
-#ubuntu-vault-sep:
-#  stage: deploy-gce-part1
-#  <<: *job
-#  <<: *gce
-#  variables:
-#    <<: *gce_variables
-#    <<: *ubuntu_vault_sep_variables
-#  when: manual
-#  except: ['triggers']
-#  only: ['master', /^pr-.*$/]
+ubuntu-vault-sep:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_vault_sep_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
 
 ubuntu-flannel-rbac-sep:
   stage: deploy-gce-special

roles/etcd/tasks/gen_certs_vault.yml

@@ -58,6 +58,9 @@
         [
         {%- for host in groups.etcd  -%}
         "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- if hostvars[host]['ip'] is defined -%}
+        "{{ hostvars[host]['ip'] }}",
+        {%- endif -%}
         {%- endfor -%}
         "127.0.0.1","::1"
         ]
@@ -81,6 +84,9 @@
         [
         {%- for host in etcd_node_cert_hosts -%}
         "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- if hostvars[host]['ip'] is defined -%}
+        "{{ hostvars[host]['ip'] }}",
+        {%- endif -%}
         {%- endfor -%}
         "127.0.0.1","::1"
         ]

roles/kubernetes/secrets/tasks/gen_certs_vault.yml

@@ -10,16 +10,16 @@
 - name: gen_certs_vault | Read in the local credentials
   command: cat /etc/vault/roles/kube/userpass
   register: kube_vault_creds_cat
-  when: inventory_hostname == groups['k8s-cluster']|first
+  delegate_to: "{{ groups['k8s-cluster'][0] }}"
 
 - name: gen_certs_vault | Set facts for read Vault Creds
   set_fact:
-    kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}"
-  when: inventory_hostname == groups['k8s-cluster']|first
+    kube_vault_creds: "{{ kube_vault_creds_cat.stdout|from_json }}"
+  delegate_to: "{{ groups['k8s-cluster'][0] }}"
 
 - name: gen_certs_vault | Log into Vault and obtain an token
   uri:
-    url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
+    url: "{{ hostvars[groups['vault'][0]]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
     headers:
       Accept: application/json
       Content-Type: application/json
@@ -28,14 +28,15 @@
     body:
       password: "{{ kube_vault_creds.password }}"
   register: kube_vault_login_result
-  when: inventory_hostname == groups['k8s-cluster']|first
+  delegate_to: "{{ groups['k8s-cluster'][0] }}"
 
 - name: gen_certs_vault | Set fact for Vault API token
   set_fact:
     kube_vault_headers:
         Accept: application/json
         Content-Type: application/json
-        X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}"
+        X-Vault-Token: "{{ kube_vault_login_result.get('json',{}).get('auth', {}).get('client_token') }}"
+  run_once: true
 
 # Issue certs to kube-master nodes
 - include: ../../../vault/tasks/shared/issue_cert.yml
@@ -67,13 +68,16 @@
         [
         {%- for host in groups['kube-master']  -%}
         "{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+        {%- if hostvars[host]['ip'] is defined -%}
+        "{{ hostvars[host]['ip'] }}",
+        {%- endif -%}
         {%- endfor -%}
         "127.0.0.1","::1","{{ kube_apiserver_ip }}"
         ]
     issue_cert_path: "{{ item }}"
     issue_cert_role: kube
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
-  with_items: "{{ kube_api_certs_needed|d([]) }}"
+  with_items: "{{ kube_master_components_certs_needed|d([]) }}"
   when: inventory_hostname in groups['kube-master']
 
 # Issue node certs to k8s-cluster nodes
@@ -89,3 +93,16 @@
     issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
   with_items: "{{ kube_node_certs_needed|d([]) }}"
   when: inventory_hostname in groups['k8s-cluster']
+
+- include: ../../../vault/tasks/shared/issue_cert.yml
+  vars:
+    issue_cert_copy_ca: "{{ item == kube_proxy_certs_needed|first }}"
+    issue_cert_file_group: "{{ kube_cert_group }}"
+    issue_cert_file_owner: kube
+    issue_cert_headers: "{{ kube_vault_headers }}"
+    issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
+    issue_cert_path: "{{ item }}"
+    issue_cert_role: kube
+    issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+  with_items: "{{ kube_proxy_certs_needed|d([]) }}"
+  when: inventory_hostname in groups['k8s-cluster']

roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml

@@ -27,24 +27,24 @@
 
 - include: ../../../vault/tasks/shared/sync_file.yml
   vars:
-    sync_file: "apiserver.pem"
+    sync_file: "{{ item }}"
     sync_file_dir: "{{ kube_cert_dir }}"
     sync_file_group: "{{ kube_cert_group }}"
     sync_file_hosts: "{{ groups['kube-master'] }}"
     sync_file_is_cert: true
     sync_file_owner: kube
+  with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem"]
 
-- name: sync_kube_master_certs | Set facts for apiserver sync_file results
+- name: sync_kube_master_certs | Set facts for kube master components sync_file results
   set_fact:
-    kube_api_certs_needed: "{{ item.path }}"
+    kube_master_components_certs_needed: "{{ kube_master_components_certs_needed|d([]) + [item.path] }}"
   with_items: "{{ sync_file_results|d([]) }}"
-  when: item.no_srcs
+  when: item.no_srcs|bool
 
-- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert
+- name: sync_kube_master_certs | Unset sync_file_results after kube master components cert
   set_fact:
     sync_file_results: []
 
-
 - include: ../../../vault/tasks/shared/sync_file.yml
   vars:
     sync_file: ca.pem

roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml

@@ -36,3 +36,27 @@
 - name: sync_kube_node_certs | Unset sync_file_results after ca.pem
   set_fact:
     sync_file_results: []
+
+- name: sync_kube_node_certs | Create list of needed kube-proxy certs
+  set_fact:
+    kube_proxy_cert_list: "{{ kube_proxy_cert_list|default([]) + ['kube-proxy-' + item + '.pem'] }}"
+  with_items: "{{ groups['k8s-cluster'] }}"
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+  vars:
+    sync_file: "{{ item }}"
+    sync_file_dir: "{{ kube_cert_dir }}"
+    sync_file_group: "{{ kube_cert_group }}"
+    sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+    sync_file_owner: kube
+  with_items: "{{ kube_proxy_cert_list|default([]) }}"
+
+- name: sync_kube_node_certs | Set facts for kube-proxy sync_file results
+  set_fact:
+    kube_proxy_certs_needed: "{{ kube_proxy_certs_needed|default([]) + [item.path] }}"
+  with_items: "{{ sync_file_results|d([]) }}"
+  when: item.no_srcs|bool
+
+- name: sync_kube_node_certs | Unset sync_file_results after kube proxy certs
+  set_fact:
+    sync_file_results: []

roles/reset/tasks/main.yml

@@ -90,6 +90,7 @@
     - /usr/local/share/ca-certificates/etcd-ca.crt
     - /etc/ssl/certs/kube-ca.pem
     - /etc/ssl/certs/etcd-ca.pem
+    - /etc/vault
     - /var/log/pods/
     - "{{ bin_dir }}/kubelet"
     - "{{ bin_dir }}/etcd-scripts"
@@ -102,7 +103,6 @@
     - "{{ bin_dir }}/weave"
   tags: ['files']
 
-
 - name: reset | remove dns settings from dhclient.conf
   blockinfile:
     dest: "{{ item }}"

roles/vault/defaults/main.yml

@@ -54,7 +54,7 @@ vault_download_vars:
   unarchive: true
   url: "{{ vault_download_url }}"
   version: "{{ vault_version }}"
-vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379"
+vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ip']|d(hostvars[groups.etcd[0]]['ansible_default_ipv4']['address']) }}:2379"
 vault_image_repo: "vault"
 vault_image_tag: "{{ vault_version }}"
 vault_log_dir: "/var/log/vault"

roles/vault/tasks/shared/issue_cert.yml

@@ -5,6 +5,7 @@
 # Vars:
 #   issue_cert_alt_name:    Requested Subject Alternative Names, in a list.
 #   issue_cert_common_name: Common Name included in the cert
+#   issue_cert_copy_ca:     Copy issuing CA cert needed
 #   issue_cert_dir_mode:    Mode of the placed cert directory
 #   issue_cert_file_group:  Group of the placed cert file and directory
 #   issue_cert_file_mode:   Mode of the placed cert file