From 3f274115b0e186c494a35433bf4e25cda4dc92dc Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Tue, 27 Dec 2016 14:02:45 +0300
Subject: [PATCH] Generate individual certificates for k8s hosts
---
.../manifests/kube-apiserver.manifest.j2 | 6 +--
.../kube-controller-manager.manifest.j2 | 2 +-
.../node/templates/node-kubeconfig.yaml.j2 | 4 +-
roles/kubernetes/secrets/files/make-ssl.sh | 47 ++++++++++++------
.../kubernetes/secrets/tasks/check-certs.yml | 33 +++++++++----
roles/kubernetes/secrets/tasks/gen_certs.yml | 48 +++++++++++++++----
6 files changed, 101 insertions(+), 39 deletions(-)
diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index c05030697..fe9a49fa8 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -35,10 +35,10 @@ spec:
- --service-node-port-range={{ kube_apiserver_node_port_range }}
- --client-ca-file={{ kube_cert_dir }}/ca.pem
- --basic-auth-file={{ kube_users_dir }}/known_users.csv
- - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
- - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+ - --tls-cert-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}.pem
+ - --tls-private-key-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}-key.pem
- --token-auth-file={{ kube_token_dir }}/known_tokens.csv
- - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
+ - --service-account-key-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}-key.pem
- --secure-port={{ kube_apiserver_port }}
- --insecure-port={{ kube_apiserver_insecure_port }}
{% if kube_api_runtime_config is defined %}
diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
index 49dd05ba8..c604587af 100644
--- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2
@@ -23,7 +23,7 @@ spec:
- controller-manager
- --master={{ kube_apiserver_endpoint }}
- --leader-elect=true
- - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+ - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-{{ inventory_hostname }}-key.pem
- --root-ca-file={{ kube_cert_dir }}/ca.pem
- --cluster-signing-cert-file={{ kube_cert_dir }}/ca.pem
- --cluster-signing-key-file={{ kube_cert_dir }}/ca-key.pem
diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2
index e1593303d..0951f2342 100644
--- a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2
+++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2
@@ -8,8 +8,8 @@ clusters:
users:
- name: kubelet
user:
- client-certificate: {{ kube_cert_dir }}/node.pem
- client-key: {{ kube_cert_dir }}/node-key.pem
+ client-certificate: {{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem
+ client-key: {{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem
contexts:
- context:
cluster: local
diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index f90fb7e8b..422bde6cf 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -27,8 +27,11 @@ Usage : $(basename $0) -f <config> [-d <ssldir>]
-f | --config : Openssl configuration file
-d | --ssldir : Directory where the certificates will be installed
- ex :
- $(basename $0) -f openssl.conf -d /srv/ssl
+ Environmental variables MASTERS and HOSTS should be set to generate keys
+ for each host.
+
+ ex :
+ MASTERS=node1 HOSTS="node1 node2" $(basename $0) -f openssl.conf -d /srv/ssl
EOF
}
@@ -61,21 +64,37 @@ cd "${tmpdir}"
mkdir -p "${SSLDIR}"
# Root CA
-openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
-openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
+if [ -e "$SSLDIR/ca-key.pem" ]; then
+ # Reuse existing CA
+ cp $SSLDIR/{ca.pem,ca-key.pem} .
+else
+ openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+ openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
+fi
+
+if [ -n "$MASTERS" ]; then
+ for host in $MASTERS; do
+ # kube-apiserver key
+ openssl genrsa -out apiserver-${host}-key.pem 2048 > /dev/null 2>&1
+ openssl req -new -key apiserver-${host}-key.pem -out apiserver-${host}.csr -subj "/CN=kube-apiserver-${host}" -config ${CONFIG} > /dev/null 2>&1
+ openssl x509 -req -in apiserver-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver-${host}.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+ cat ca.pem >> apiserver-${host}.pem
-# Apiserver
-openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
-openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
-openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
-cat ca.pem >> apiserver.pem
+ # admin key
+ openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
+ openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=kube-admin-${host}" > /dev/null 2>&1
+ openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 365 > /dev/null 2>&1
+ done
+fi
# Nodes and Admin
-for i in node admin; do
- openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
- openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
- openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 > /dev/null 2>&1
-done
+if [ -n "$HOSTS" ]; then
+ for host in $HOSTS; do
+ openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
+ openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=kube-node-${host}" > /dev/null 2>&1
+ openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 365 > /dev/null 2>&1
+ done
+fi
# Install certs
mv *.pem ${SSLDIR}/
diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml
index 97c6f7226..aa3e47754 100644
--- a/roles/kubernetes/secrets/tasks/check-certs.yml
+++ b/roles/kubernetes/secrets/tasks/check-certs.yml
@@ -1,36 +1,49 @@
---
- name: "Check_certs | check if the certs have already been generated on first master"
stat:
- path: "{{ kube_cert_dir }}/ca.pem"
+ path: "{{ kube_cert_dir }}/{{ item }}"
delegate_to: "{{groups['kube-master'][0]}}"
register: kubecert_master
run_once: true
+ with_items: >-
+ ['ca.pem',
+ {% for host in groups['k8s-cluster'] %}
+ 'node-{{ host }}-key.pem'
+ {% if not loop.last %}{{','}}{% endif %}
+ {% endfor %}]
-- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
+- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs', and 'secret_changed' to false"
set_fact:
sync_certs: false
gen_certs: false
+ secret_changed: false
-- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
+- name: "Check_certs | Set 'gen_certs' to true"
set_fact:
gen_certs: true
- when: not kubecert_master.stat.exists
+ when: "not {{ item.stat.exists }}"
run_once: true
+ with_items: "{{ kubecert_master.results }}"
- name: "Check certs | check if a cert already exists"
stat:
- path: "{{ kube_cert_dir }}/ca.pem"
+ path: "{{ kube_cert_dir }}/{{ item }}"
register: kubecert
+ with_items:
+ - ca.pem
+ - node-{{ inventory_hostname }}-key.pem
- name: "Check_certs | Set 'sync_certs' to true"
set_fact:
sync_certs: true
when: >-
{%- set certs = {'sync': False} -%}
- {%- for server in play_hosts
- if (not hostvars[server].kubecert.stat.exists|default(False)) or
- (hostvars[server].kubecert.stat.checksum|default('') != kubecert_master.stat.checksum|default('')) -%}
- {%- set _ = certs.update({'sync': True}) -%}
+ {%- for host in groups['k8s-cluster'] %}
+ {% if host == inventory_hostname %}
+ {% if (not kubecert.results[0].stat.exists|default(False)) or
+ (kubecert.results[1].stat.checksum|default('') != kubecert_master.results[loop.index].stat.checksum|default('')) -%}
+ {%- set _ = certs.update({'sync': True}) -%}
+ {% endif %}
+ {% endif %}
{%- endfor -%}
{{ certs.sync }}
- run_once: true
diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml
index 22116fd3d..bd652e340 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs.yml
@@ -18,35 +18,65 @@
- name: Gen_certs | run cert generation script
command: "{{ kube_script_dir }}/make-ssl.sh -f {{ kube_config_dir }}/openssl.conf -d {{ kube_cert_dir }}"
+ environment:
+ - MASTERS: "{% for m in groups['kube-master'] %}
+ {% if hostvars[m].sync_certs|default(true) %}
+ {{ m }}
+ {% endif %}
+ {% endfor %}"
+ - HOSTS: "{% for h in groups['k8s-cluster'] %}
+ {% if hostvars[h].sync_certs|default(true) %}
+ {{ h }}
+ {% endif %}
+ {% endfor %}"
run_once: yes
delegate_to: "{{groups['kube-master'][0]}}"
when: gen_certs|default(false)
notify: set secret_changed
- set_fact:
- master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'apiserver-key.pem', 'apiserver.pem']
- node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
+ all_master_certs: "['ca-key.pem',
+ {% for node in groups['kube-master'] %}
+ 'admin-{{ node }}.pem',
+ 'admin-{{ node }}-key.pem',
+ 'apiserver-{{ node }}.pem',
+ 'apiserver-{{ node }}-key.pem',
+ {% endfor %}]"
+ my_master_certs: ['ca-key.pem',
+ 'admin-{{ inventory_hostname }}.pem',
+ 'admin-{{ inventory_hostname }}-key.pem',
+ 'apiserver-{{ inventory_hostname }}.pem',
+ 'apiserver-{{ inventory_hostname }}-key.pem'
+ ]
+ all_node_certs: "['ca.pem',
+ {% for node in groups['k8s-cluster'] %}
+ 'node-{{ node }}.pem',
+ 'node-{{ node }}-key.pem',
+ {% endfor %}]"
+ my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem']
tags: facts
- name: Gen_certs | Gather master certs
- shell: "tar cfz - -C {{ kube_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+ shell: "tar cfz - -C {{ kube_cert_dir }} {{ my_master_certs|join(' ') }} {{ all_node_certs|join(' ') }} | base64 --wrap=0"
register: master_cert_data
delegate_to: "{{groups['kube-master'][0]}}"
- run_once: true
- when: sync_certs|default(false)
+ when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and
+ inventory_hostname != groups['kube-master'][0]
- name: Gen_certs | Gather node certs
- shell: "tar cfz - -C {{ kube_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+ shell: "tar cfz - -C {{ kube_cert_dir }} {{ my_node_certs|join(' ') }} | base64 --wrap=0"
register: node_cert_data
delegate_to: "{{groups['kube-master'][0]}}"
- run_once: true
- when: sync_certs|default(false)
+ when: inventory_hostname in groups['kube-node'] and
+ sync_certs|default(false) and
+ inventory_hostname != groups['kube-master'][0]
- name: Gen_certs | Copy certs on masters
shell: "echo '{{master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}"
changed_when: false
when: inventory_hostname in groups['kube-master'] and sync_certs|default(false) and
inventory_hostname != groups['kube-master'][0]
+ notify: set secret_changed
- name: Gen_certs | Copy certs on nodes
shell: "echo '{{node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ kube_cert_dir }}"
@@ -54,6 +84,7 @@
when: inventory_hostname in groups['kube-node'] and
sync_certs|default(false) and
inventory_hostname != groups['kube-master'][0]
+ notify: set secret_changed
- name: Gen_certs | check certificate permissions
file:
@@ -93,4 +124,3 @@
- name: Gen_certs | update ca-certificates (RedHat)
command: update-ca-trust extract
when: kube_ca_cert.changed and ansible_os_family == "RedHat"
-
--
GitLab