Individual etcd ssl certs

Includes hooks for triggering calico, kubelet, and kube-apiserver restarts
if etcd certs changed.

roles/etcd/files/make-ssl-etcd.sh

@@ -16,7 +16,6 @@
 
 set -o errexit
 set -o pipefail
-
 usage()
 {
     cat << EOF
@@ -61,20 +60,43 @@ cd "${tmpdir}"
 mkdir -p "${SSLDIR}"
 
 # Root CA
-openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
-openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+if [ -e "$SSLDIR/ca-key.pem" ]; then
+    # Reuse existing CA
+    cp $SSLDIR/{ca.pem,ca-key.pem} .
+else
+    openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+    openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+fi
 
 # ETCD member
-openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
-openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
-openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+if [ -n "$MASTERS" ]; then
+    for host in $MASTERS; do
+        openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1
+        openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${host}" -config ${CONFIG} > /dev/null 2>&1
+        openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+    done
+else
+    openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
+    openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+fi
 
-# Nodes and Admin
-for i in node admin; do
-    openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
-    openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
-    openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
-done
+# Node and admin keys
+if [ -n "$HOSTS" ]; then
+    for host in $HOSTS; do
+        for i in node admin; do
+            openssl genrsa -out ${i}-${host}-key.pem 2048 > /dev/null 2>&1
+            openssl req -new -key ${i}-${host}-key.pem -out ${i}-${host}.csr -subj "/CN=kube-${i}-${host}" > /dev/null 2>&1
+            openssl x509 -req -in ${i}-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}-${host}.pem -days 365 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+        done
+     done
+else
+    for i in node admin; do
+        openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+        openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+        openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
+    done
+fi
 
 # Install certs
 mv *.pem ${SSLDIR}/

roles/etcd/tasks/check_certs.yml

 ---
-- name: "Check_certs | check if the certs have already been generated on first master"
+- name: "Check_certs | check if all certs have already been generated on first master"
   stat:
-    path: "{{ etcd_cert_dir }}/ca.pem"
+    path: "{{ etcd_cert_dir }}/{{ item }}"
   delegate_to: "{{groups['etcd'][0]}}"
   register: etcdcert_master
   run_once: true
+  with_items: >-
+       ['ca.pem',
+       {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|unique %}
+       {% for host in all_etcd_hosts %}
+       'node-{{ host }}-key.pem'
+       {% if not loop.last %}{{','}}{% endif %}
+       {% endfor %}]
 
-- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
+- name: "Check_certs | Set default value for 'sync_certs', 'gen_certs' and 'etcd_secret_changed' to false"
   set_fact:
     sync_certs: false
     gen_certs: false
+    etcd_secret_changed: false
 
-- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
+- name: "Check_certs | Set 'gen_certs' to true"
   set_fact:
     gen_certs: true
-  when: not etcdcert_master.stat.exists
+  when: "not {{item.stat.exists}}"
   run_once: true
+  with_items: "{{etcdcert_master.results}}"
 
 - name: "Check certs | check if a cert already exists"
   stat:
-    path: "{{ etcd_cert_dir }}/ca.pem"
+    path: "{{ etcd_cert_dir }}/node-{{inventory_hostname}}-key.pem"
   register: etcdcert
+  with_items:
+    - ca.pem
+    - node-{{ inventory_hostname }}-key.pem
 
 - name: "Check_certs | Set 'sync_certs' to true"
   set_fact:
     sync_certs: true
   when: >-
       {%- set certs = {'sync': False} -%}
-      {%- for server in play_hosts
-         if (not hostvars[server].etcdcert.stat.exists|default(False)) or
-         (hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%}
-         {%- set _ = certs.update({'sync': True}) -%}
+      {% set all_etcd_hosts = groups['k8s-cluster']|union(groups['etcd'])|unique %}
+      {% for host in all_etcd_hosts %}
+        {% if host == inventory_hostname %}
+          {% if (not etcdcert.results[0].stat.exists|default(False)) or
+              (etcdcert.results[1].stat.checksum|default('') != etcdcert_master.results[loop.index].stat.checksum|default('')) -%}
+            {%- set _ = certs.update({'sync': True}) -%}
+          {% endif %}
+        {% endif %}
       {%- endfor -%}
       {{ certs.sync }}
-  run_once: true

roles/etcd/tasks/gen_certs.yml

@@ -34,29 +34,56 @@
 
 - name: Gen_certs | run cert generation script
   command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+  environment:
+    - MASTERS: "{% for m in groups['etcd'] %}
+                  {% if hostvars[m].sync_certs|default(false) %}
+                    {{ m }}
+                  {% endif %}
+                {% endfor %}"
+    - HOSTS: "{% for h in groups['k8s-cluster'] %}
+                {% if hostvars[h].sync_certs|default(false) %}
+                    {{ h }}
+                {% endif %}
+              {% endfor %}"
   run_once: yes
   delegate_to: "{{groups['etcd'][0]}}"
   when: gen_certs|default(false)
   notify: set etcd_secret_changed
 
 - set_fact:
-    master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
-    node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
+    all_master_certs: "['ca-key.pem',
+                      {% for node in groups['etcd'] %}
+                      'admin-{{ node }}.pem',
+                      'admin-{{ node }}-key.pem',
+                      'member-{{ node }}.pem',
+                      'member-{{ node }}-key.pem',
+                      {% endfor %}]"
+    my_master_certs: ['ca-key.pem',
+                     'admin-{{ inventory_hostname }}.pem',
+                     'admin-{{ inventory_hostname }}-key.pem',
+                     'member-{{ inventory_hostname }}.pem',
+                     'member-{{ inventory_hostname }}-key.pem'
+                     ]
+    all_node_certs: "['ca.pem',
+                    {% for node in groups['k8s-cluster'] %}
+                    'node-{{ node }}.pem',
+                    'node-{{ node }}-key.pem',
+                    {% endfor %}]"
+    my_node_certs: ['ca.pem', 'node-{{ inventory_hostname }}.pem', 'node-{{ inventory_hostname }}-key.pem']
   tags: facts
 
 - name: Gen_certs | Gather etcd master certs
-  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0"
+  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ my_master_certs|join(' ') }} {{ all_node_certs|join(' ') }}| base64 --wrap=0"
   register: etcd_master_cert_data
   delegate_to: "{{groups['etcd'][0]}}"
-  run_once: true
+  #run_once: true
   when: sync_certs|default(false)
   notify: set etcd_secret_changed
 
 - name: Gen_certs | Gather etcd node certs
-  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+  shell: "tar cfz - -C {{ etcd_cert_dir }} {{ my_node_certs|join(' ') }} | base64 --wrap=0"
   register: etcd_node_cert_data
   delegate_to: "{{groups['etcd'][0]}}"
-  run_once: true
   when: sync_certs|default(false)
   notify: set etcd_secret_changed
 

roles/etcd/tasks/main.yml

@@ -22,10 +22,11 @@
     enabled: yes
   when: is_etcd_master
 
-- name: Restart etcd if binary changed
+- name: Restart etcd if binary or certs changed
   command: /bin/true
   notify: restart etcd
   when: etcd_deployment_type == "host" and etcd_copy.stdout_lines and is_etcd_master
+    or etcd_secret_changed|default(false)
 
 # Reload systemd before starting service
 - meta: flush_handlers

roles/etcd/tasks/pre_upgrade.yml

@@ -34,10 +34,6 @@
   command: "docker rm -f {{item}}"
   with_items: "{{etcd_proxy_container.stdout_lines}}"
 
-- name: "Pre-upgrade | remove etcd-proxy if it exists"
-  command: "docker rm -f {{item}}"
-  with_items: "{{etcd_proxy_container.stdout_lines}}"
-
 - name: "Pre-upgrade | check if member list is non-SSL"
   command: "{{ bin_dir }}/etcdctl --no-sync --peers={{ etcd_access_addresses | regex_replace('https','http') }} member list"
   register: etcd_member_list
@@ -53,4 +49,3 @@
     awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash
   run_once: true
   when: 'etcd_member_list is defined and "http://" in etcd_member_list.stdout'
-

roles/etcd/templates/etcd.j2

@@ -13,9 +13,9 @@ ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
 
 # TLS settings
 ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
-ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem
-ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
 ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
-ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem
-ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-{{ inventory_hostname }}-key.pem
 ETCD_PEER_CLIENT_CERT_AUTH=true

roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2

@@ -31,9 +31,9 @@ spec:
             - name: ETCD_CA_CERT_FILE
               value: "{{ etcd_cert_dir }}/ca.pem"
             - name: ETCD_CERT_FILE
-              value: "{{ etcd_cert_dir }}/node.pem"
+              value: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
             - name: ETCD_KEY_FILE
-              value: "{{ etcd_cert_dir }}/node-key.pem"
+              value: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
             # Location of the Kubernetes API - this shouldn't need to be
             # changed so long as it is used in conjunction with
             # CONFIGURE_ETC_HOSTS="true".

roles/kubernetes/master/tasks/pre-upgrade.yml

@@ -27,3 +27,23 @@
     - /etc/systemd/system/kube-apiserver.service
     - /etc/init.d/kube-apiserver
   tags: kube-apiserver
+
+- name: "Pre-upgrade | See if kube-apiserver manifest exists"
+  stat:
+    path: /etc/kubernetes/manifests/kube-apiserver.manifest
+  register: kube_apiserver_manifest
+  when: secret_changed|default(false) or etcd_secret_changed|default(false)
+
+- name: "Pre-upgrade | Write invalid image to kube-apiserver manifest if secrets were changed"
+  replace:
+    dest: /etc/kubernetes/manifests/kube-apiserver.manifest
+    regexp: '(\s+)image:\s+.*?$'
+    replace: '\1image: kill.apiserver.using.fake.image.in:manifest'
+  register: kube_apiserver_manifest_replaced
+  when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists
+
+- name: "Pre-upgrade | Pause while waiting for kubelet to delete kube-apiserver pod"
+  pause: seconds=20
+  when: (secret_changed|default(false) or etcd_secret_changed|default(false)) and kube_apiserver_manifest.stat.exists
+  tags: kube-apiserver
+

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -5,6 +5,7 @@ metadata:
   namespace: {{system_namespace}}
   labels:
     k8s-app: kube-apiserver
+    kargo: v2
 spec:
   hostNetwork: true
   containers:
@@ -18,8 +19,8 @@ spec:
     - --etcd-servers={{ etcd_access_endpoint }}
     - --etcd-quorum-read=true
     - --etcd-cafile={{ etcd_cert_dir }}/ca.pem
-    - --etcd-certfile={{ etcd_cert_dir }}/node.pem
-    - --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem
+    - --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem
+    - --etcd-keyfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem
     - --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
     - --apiserver-count={{ kube_apiserver_count }}
     - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota

roles/network_plugin/calico/tasks/main.yml

@@ -27,8 +27,8 @@
     force: yes
   with_items:
     - {s: "ca.pem", d: "ca_cert.crt"}
-    - {s: "node.pem", d: "cert.crt"}
-    - {s: "node-key.pem", d: "key.pem"}
+    - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
+    - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
 
 - name: Calico | Install calicoctl container script
   template:
@@ -72,8 +72,8 @@
   command: |-
     curl \
       --cacert {{ etcd_cert_dir }}/ca.pem \
-      --cert {{ etcd_cert_dir}}/admin.pem \
-      --key {{ etcd_cert_dir }}/admin-key.pem \
+      --cert {{ etcd_cert_dir}}/admin-{{ groups['etcd'][0] }}.pem \
+      --key {{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem \
       https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
   register: calico_conf
   delegate_to: "{{groups['etcd'][0]}}"
@@ -131,8 +131,8 @@
   command: |-
     curl \
       --cacert {{ etcd_cert_dir }}/ca.pem \
-      --cert {{ etcd_cert_dir}}/admin.pem \
-      --key {{ etcd_cert_dir }}/admin-key.pem \
+      --cert {{ etcd_cert_dir}}/admin-{{ groups['etcd'][0] }}.pem \
+      --key {{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem \
       https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
   register: calico_pools_raw
   delegate_to: "{{groups['etcd'][0]}}"
@@ -191,6 +191,11 @@
   when: ansible_service_mgr in ["sysvinit","upstart"] and ansible_os_family == "RedHat"
   notify: restart calico-node
 
+- name: Calico | Restart calico-node if secrets changed
+  command: /bin/true
+  when: secret_changed|default(false) or etcd_secret_changed|default(false)
+  notify: restart calico-node
+
 - meta: flush_handlers
 
 - name: Calico | Enable calico-node

roles/network_plugin/calico/templates/cni-calico.conf.j2

@@ -5,8 +5,8 @@
 {% endif %}
   "type": "calico",
   "etcd_endpoints": "{{ etcd_access_endpoint }}",
-  "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
-  "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
+  "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem",
+  "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem",
   "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
   "log_level": "info",
   "ipam": {

roles/network_plugin/canal/tasks/main.yml

@@ -21,8 +21,8 @@
     force: yes
   with_items:
     - {s: "ca.pem", d: "ca_cert.crt"}
-    - {s: "node.pem", d: "cert.crt"}
-    - {s: "node-key.pem", d: "key.pem"}
+    - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
+    - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
 
 - name: Canal | Set Flannel etcd configuration
   command: |-

roles/network_plugin/flannel/templates/flannel-pod.yml

@@ -22,7 +22,7 @@
         command:
           - "/bin/sh"
           - "-c"
-          - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
+          - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -etcd-keyfile {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
         ports:
           - hostPort: 10253
             containerPort: 10253