From c1ba8e1b3a15453303c69f08dd74e6598a0c195c Mon Sep 17 00:00:00 2001
From: Lovro Seder <vrovro@gmail.com>
Date: Thu, 3 Sep 2020 16:25:41 +0200
Subject: [PATCH] Rotate kubelet server certificate. (#6453)
* Rotate kubelet server certificate.
* CI test kubelet server cert rotation
* Approve kubelet serving certificates in tests.
---
docs/vars.md | 5 ++++
.../templates/kubeadm-config.v1beta2.yaml.j2 | 5 +++-
.../templates/kubelet-config.v1beta1.yaml.j2 | 3 +++
roles/kubespray-defaults/defaults/main.yaml | 2 ++
...t_centos7-flannel-containerd-addons-ha.yml | 1 +
tests/testcases/030_check-network.yml | 24 +++++++++++++++++++
6 files changed, 39 insertions(+), 1 deletion(-)
diff --git a/docs/vars.md b/docs/vars.md
index db672a389..d46d789ce 100644
--- a/docs/vars.md
+++ b/docs/vars.md
@@ -128,6 +128,11 @@ Stack](https://github.com/kubernetes-sigs/kubespray/blob/master/docs/dns-stack.m
to match Docker configuration.
* *kubelet_rotate_certificates* - Auto rotate the kubelet client certificates by requesting new certificates
from the kube-apiserver when the certificate expiration approaches.
+* *kubelet_rotate_server_certificates* - Auto rotate the kubelet server certificates by requesting new certificates
+ from the kube-apiserver when the certificate expiration approaches.
+ **Note** that server certificates are **not** approved automatically. Approve them manually
+ (`kubectl get csr`, `kubectl certificate approve`) or implement custom approving controller like
+ [kubelet-rubber-stamp](https://github.com/kontena/kubelet-rubber-stamp).
* *node_labels* - Labels applied to nodes via kubelet --node-labels parameter.
For example, labels can be set in the inventory as variables or more widely in group_vars.
*node_labels* can be defined either as a dict or a comma-separated labels string:
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2
index 2e3d20a24..edbdf9fe2 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2
@@ -198,7 +198,10 @@ apiServer:
{% endif %}
{% if event_ttl_duration is defined %}
event-ttl: {{ event_ttl_duration }}
-{%endif%}
+{% endif %}
+{% if kubelet_rotate_server_certificates %}
+ kubelet-certificate-authority: {{ kube_cert_dir }}/ca.crt
+{% endif %}
{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
index f2341e3b7..b07775d56 100644
--- a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
+++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2
@@ -34,6 +34,9 @@ clusterDomain: {{ dns_domain }}
{% if kubelet_rotate_certificates|bool %}
rotateCertificates: true
{% endif %}
+{% if kubelet_rotate_server_certificates|bool %}
+serverTLSBootstrap: true
+{% endif %}
{# DNS settings for kubelet #}
{% if enable_nodelocaldns %}
{% set kubelet_cluster_dns = [nodelocaldns_ip] %}
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index dcbec2586..9bc38e4ae 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -394,6 +394,8 @@ kubelet_authorization_mode_webhook: true
# kubelet uses certificates for authenticating to the Kubernetes API
# Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration
kubelet_rotate_certificates: true
+# kubelet can also request a new server certificate from the Kubernetes API
+kubelet_rotate_server_certificates: false
## List of key=value pairs that describe feature gates for
## the k8s cluster.
diff --git a/tests/files/packet_centos7-flannel-containerd-addons-ha.yml b/tests/files/packet_centos7-flannel-containerd-addons-ha.yml
index 09174515e..8ca0feaee 100644
--- a/tests/files/packet_centos7-flannel-containerd-addons-ha.yml
+++ b/tests/files/packet_centos7-flannel-containerd-addons-ha.yml
@@ -25,6 +25,7 @@ metrics_server_kubelet_insecure_tls: true
kube_token_auth: true
kube_basic_auth: true
enable_nodelocaldns: false
+kubelet_rotate_server_certificates: true
kube_oidc_url: https://accounts.google.com/.well-known/openid-configuration
kube_oidc_client_id: kubespray-example
diff --git a/tests/testcases/030_check-network.yml b/tests/testcases/030_check-network.yml
index 73e5de22e..c0ca036d8 100644
--- a/tests/testcases/030_check-network.yml
+++ b/tests/testcases/030_check-network.yml
@@ -15,6 +15,30 @@
bin_dir: "/usr/local/bin"
when: not ansible_os_family in ["Flatcar Container Linux by Kinvolk"]
+ - name: Approve kubelet serving certificates
+ block:
+
+ - name: Get certificate signing requests
+ command: "{{ bin_dir }}/kubectl get csr -o name"
+ register: get_csr
+ changed_when: false
+
+ - name: Check there are csrs
+ assert:
+ that: get_csr.stdout_lines | length > 0
+ fail_msg: kubelet_rotate_server_certificates is {{ kubelet_rotate_server_certificates }} but no csr's found
+
+ - name: Approve certificates
+ command: "{{ bin_dir }}/kubectl certificate approve {{ get_csr.stdout_lines | join(' ') }}"
+ register: certificate_approve
+ when: get_csr.stdout_lines | length > 0
+ changed_when: certificate_approve.stdout
+
+ - debug:
+ msg: "{{ certificate_approve.stdout.split('\n') }}"
+
+ when: kubelet_rotate_server_certificates | default(false)
+
- name: Create test namespace # noqa 301 305
shell: "{{ bin_dir }}/kubectl create namespace test"
--
GitLab