From 2ba285a544a9d9fc342166e472d2fde6b48aeda3 Mon Sep 17 00:00:00 2001
From: Maxim Krasilnikov <mak.krasilnikov@gmail.com>
Date: Sun, 20 Aug 2017 13:53:58 +0300
Subject: [PATCH] Fixed deploy cluster with vault cert manager (#1548)
* Added custom ips to etcd vault distributed certificates
* Added custom ips to kube-master vault distributed certificates
* Added comment about issue_cert_copy_ca var in vault/issue_cert role file
* Generate kube-proxy, controller-manager and scheduler certificates by vault
* Revert "Disable vault from CI (#1546)"
This reverts commit 781f31d2b81b98c8aa6cfda0897d5bd4d966107c.
* Fixed upgrade cluster with vault cert manager
* Remove vault dir in reset playbook
---
.gitlab-ci.yml | 38 +++++++++----------
roles/etcd/tasks/gen_certs_vault.yml | 6 +++
.../secrets/tasks/gen_certs_vault.yml | 31 +++++++++++----
.../secrets/tasks/sync_kube_master_certs.yml | 12 +++---
.../secrets/tasks/sync_kube_node_certs.yml | 24 ++++++++++++
roles/reset/tasks/main.yml | 2 +-
roles/vault/defaults/main.yml | 2 +-
roles/vault/tasks/shared/issue_cert.yml | 1 +
8 files changed, 81 insertions(+), 35 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7a53edd83..a7e5130c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -367,15 +367,14 @@ before_script:
KUBELET_DEPLOYMENT: rkt
STARTUP_SCRIPT: ""
-#Note(mattymo): Vault deployment is broken and needs work
-#.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
-## stage: deploy-gce-part1
-# KUBE_NETWORK_PLUGIN: canal
-# CERT_MGMT: vault
-# CLOUD_IMAGE: ubuntu-1604-xenial
-# CLOUD_REGION: us-central1-b
-# CLUSTER_MODE: separate
-# STARTUP_SCRIPT: ""
+.ubuntu_vault_sep_variables: &ubuntu_vault_sep_variables
+# stage: deploy-gce-part1
+ KUBE_NETWORK_PLUGIN: canal
+ CERT_MGMT: vault
+ CLOUD_IMAGE: ubuntu-1604-xenial
+ CLOUD_REGION: us-central1-b
+ CLUSTER_MODE: separate
+ STARTUP_SCRIPT: ""
.ubuntu_flannel_rbac_variables: &ubuntu_flannel_rbac_variables
# stage: deploy-gce-special
@@ -600,17 +599,16 @@ ubuntu-rkt-sep:
except: ['triggers']
only: ['master', /^pr-.*$/]
-#Note(mattymo): Vault deployment is broken (https://github.com/kubernetes-incubator/kubespray/issues/1545)
-#ubuntu-vault-sep:
-# stage: deploy-gce-part1
-# <<: *job
-# <<: *gce
-# variables:
-# <<: *gce_variables
-# <<: *ubuntu_vault_sep_variables
-# when: manual
-# except: ['triggers']
-# only: ['master', /^pr-.*$/]
+ubuntu-vault-sep:
+ stage: deploy-gce-part1
+ <<: *job
+ <<: *gce
+ variables:
+ <<: *gce_variables
+ <<: *ubuntu_vault_sep_variables
+ when: manual
+ except: ['triggers']
+ only: ['master', /^pr-.*$/]
ubuntu-flannel-rbac-sep:
stage: deploy-gce-special
diff --git a/roles/etcd/tasks/gen_certs_vault.yml b/roles/etcd/tasks/gen_certs_vault.yml
index 75be6c37c..a0bf6cfdc 100644
--- a/roles/etcd/tasks/gen_certs_vault.yml
+++ b/roles/etcd/tasks/gen_certs_vault.yml
@@ -58,6 +58,9 @@
[
{%- for host in groups.etcd -%}
"{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+ {%- if hostvars[host]['ip'] is defined -%}
+ "{{ hostvars[host]['ip'] }}",
+ {%- endif -%}
{%- endfor -%}
"127.0.0.1","::1"
]
@@ -81,6 +84,9 @@
[
{%- for host in etcd_node_cert_hosts -%}
"{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+ {%- if hostvars[host]['ip'] is defined -%}
+ "{{ hostvars[host]['ip'] }}",
+ {%- endif -%}
{%- endfor -%}
"127.0.0.1","::1"
]
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml
index ac8e128b4..e516db0f2 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_vault.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_vault.yml
@@ -10,16 +10,16 @@
- name: gen_certs_vault | Read in the local credentials
command: cat /etc/vault/roles/kube/userpass
register: kube_vault_creds_cat
- when: inventory_hostname == groups['k8s-cluster']|first
+ delegate_to: "{{ groups['k8s-cluster'][0] }}"
- name: gen_certs_vault | Set facts for read Vault Creds
set_fact:
- kube_vault_creds: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_creds_cat']['stdout'] | from_json }}"
- when: inventory_hostname == groups['k8s-cluster']|first
+ kube_vault_creds: "{{ kube_vault_creds_cat.stdout|from_json }}"
+ delegate_to: "{{ groups['k8s-cluster'][0] }}"
- name: gen_certs_vault | Log into Vault and obtain an token
uri:
- url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
+ url: "{{ hostvars[groups['vault'][0]]['vault_leader_url'] }}/v1/auth/userpass/login/{{ kube_vault_creds.username }}"
headers:
Accept: application/json
Content-Type: application/json
@@ -28,14 +28,15 @@
body:
password: "{{ kube_vault_creds.password }}"
register: kube_vault_login_result
- when: inventory_hostname == groups['k8s-cluster']|first
+ delegate_to: "{{ groups['k8s-cluster'][0] }}"
- name: gen_certs_vault | Set fact for Vault API token
set_fact:
kube_vault_headers:
Accept: application/json
Content-Type: application/json
- X-Vault-Token: "{{ hostvars[groups['k8s-cluster']|first]['kube_vault_login_result']['json']['auth']['client_token'] }}"
+ X-Vault-Token: "{{ kube_vault_login_result.get('json',{}).get('auth', {}).get('client_token') }}"
+ run_once: true
# Issue certs to kube-master nodes
- include: ../../../vault/tasks/shared/issue_cert.yml
@@ -67,13 +68,16 @@
[
{%- for host in groups['kube-master'] -%}
"{{ hostvars[host]['ansible_default_ipv4']['address'] }}",
+ {%- if hostvars[host]['ip'] is defined -%}
+ "{{ hostvars[host]['ip'] }}",
+ {%- endif -%}
{%- endfor -%}
"127.0.0.1","::1","{{ kube_apiserver_ip }}"
]
issue_cert_path: "{{ item }}"
issue_cert_role: kube
issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
- with_items: "{{ kube_api_certs_needed|d([]) }}"
+ with_items: "{{ kube_master_components_certs_needed|d([]) }}"
when: inventory_hostname in groups['kube-master']
# Issue node certs to k8s-cluster nodes
@@ -89,3 +93,16 @@
issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
with_items: "{{ kube_node_certs_needed|d([]) }}"
when: inventory_hostname in groups['k8s-cluster']
+
+- include: ../../../vault/tasks/shared/issue_cert.yml
+ vars:
+ issue_cert_copy_ca: "{{ item == kube_proxy_certs_needed|first }}"
+ issue_cert_file_group: "{{ kube_cert_group }}"
+ issue_cert_file_owner: kube
+ issue_cert_headers: "{{ kube_vault_headers }}"
+ issue_cert_hosts: "{{ groups['k8s-cluster'] }}"
+ issue_cert_path: "{{ item }}"
+ issue_cert_role: kube
+ issue_cert_url: "{{ hostvars[groups.vault|first]['vault_leader_url'] }}"
+ with_items: "{{ kube_proxy_certs_needed|d([]) }}"
+ when: inventory_hostname in groups['k8s-cluster']
diff --git a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml
index b02120ccb..6fa861a36 100644
--- a/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml
+++ b/roles/kubernetes/secrets/tasks/sync_kube_master_certs.yml
@@ -27,24 +27,24 @@
- include: ../../../vault/tasks/shared/sync_file.yml
vars:
- sync_file: "apiserver.pem"
+ sync_file: "{{ item }}"
sync_file_dir: "{{ kube_cert_dir }}"
sync_file_group: "{{ kube_cert_group }}"
sync_file_hosts: "{{ groups['kube-master'] }}"
sync_file_is_cert: true
sync_file_owner: kube
+ with_items: ["apiserver.pem", "kube-scheduler.pem", "kube-controller-manager.pem"]
-- name: sync_kube_master_certs | Set facts for apiserver sync_file results
+- name: sync_kube_master_certs | Set facts for kube master components sync_file results
set_fact:
- kube_api_certs_needed: "{{ item.path }}"
+ kube_master_components_certs_needed: "{{ kube_master_components_certs_needed|d([]) + [item.path] }}"
with_items: "{{ sync_file_results|d([]) }}"
- when: item.no_srcs
+ when: item.no_srcs|bool
-- name: sync_kube_master_certs | Unset sync_file_results after apiserver cert
+- name: sync_kube_master_certs | Unset sync_file_results after kube master components cert
set_fact:
sync_file_results: []
-
- include: ../../../vault/tasks/shared/sync_file.yml
vars:
sync_file: ca.pem
diff --git a/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml
index 884f6c436..b97b85e17 100644
--- a/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml
+++ b/roles/kubernetes/secrets/tasks/sync_kube_node_certs.yml
@@ -36,3 +36,27 @@
- name: sync_kube_node_certs | Unset sync_file_results after ca.pem
set_fact:
sync_file_results: []
+
+- name: sync_kube_node_certs | Create list of needed kube-proxy certs
+ set_fact:
+ kube_proxy_cert_list: "{{ kube_proxy_cert_list|default([]) + ['kube-proxy-' + item + '.pem'] }}"
+ with_items: "{{ groups['k8s-cluster'] }}"
+
+- include: ../../../vault/tasks/shared/sync_file.yml
+ vars:
+ sync_file: "{{ item }}"
+ sync_file_dir: "{{ kube_cert_dir }}"
+ sync_file_group: "{{ kube_cert_group }}"
+ sync_file_hosts: "{{ groups['k8s-cluster'] }}"
+ sync_file_owner: kube
+ with_items: "{{ kube_proxy_cert_list|default([]) }}"
+
+- name: sync_kube_node_certs | Set facts for kube-proxy sync_file results
+ set_fact:
+ kube_proxy_certs_needed: "{{ kube_proxy_certs_needed|default([]) + [item.path] }}"
+ with_items: "{{ sync_file_results|d([]) }}"
+ when: item.no_srcs|bool
+
+- name: sync_kube_node_certs | Unset sync_file_results after kube proxy certs
+ set_fact:
+ sync_file_results: []
diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index 77140ba6a..e13065404 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -90,6 +90,7 @@
- /usr/local/share/ca-certificates/etcd-ca.crt
- /etc/ssl/certs/kube-ca.pem
- /etc/ssl/certs/etcd-ca.pem
+ - /etc/vault
- /var/log/pods/
- "{{ bin_dir }}/kubelet"
- "{{ bin_dir }}/etcd-scripts"
@@ -102,7 +103,6 @@
- "{{ bin_dir }}/weave"
tags: ['files']
-
- name: reset | remove dns settings from dhclient.conf
blockinfile:
dest: "{{ item }}"
diff --git a/roles/vault/defaults/main.yml b/roles/vault/defaults/main.yml
index 4a0a60982..9694599cb 100644
--- a/roles/vault/defaults/main.yml
+++ b/roles/vault/defaults/main.yml
@@ -54,7 +54,7 @@ vault_download_vars:
unarchive: true
url: "{{ vault_download_url }}"
version: "{{ vault_version }}"
-vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ansible_default_ipv4']['address'] }}:2379"
+vault_etcd_url: "https://{{ hostvars[groups.etcd[0]]['ip']|d(hostvars[groups.etcd[0]]['ansible_default_ipv4']['address']) }}:2379"
vault_image_repo: "vault"
vault_image_tag: "{{ vault_version }}"
vault_log_dir: "/var/log/vault"
diff --git a/roles/vault/tasks/shared/issue_cert.yml b/roles/vault/tasks/shared/issue_cert.yml
index 11f7abb41..4854e8b9e 100644
--- a/roles/vault/tasks/shared/issue_cert.yml
+++ b/roles/vault/tasks/shared/issue_cert.yml
@@ -5,6 +5,7 @@
# Vars:
# issue_cert_alt_name: Requested Subject Alternative Names, in a list.
# issue_cert_common_name: Common Name included in the cert
+# issue_cert_copy_ca: Copy issuing CA cert needed
# issue_cert_dir_mode: Mode of the placed cert directory
# issue_cert_file_group: Group of the placed cert file and directory
# issue_cert_file_mode: Mode of the placed cert file
--
GitLab