diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 128e8cc990e74c9b7843df9fe3a482656ab78a74..8f69afc25e05bdd2699a6a8a97e0de824c9cec25 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -111,7 +111,10 @@ kube_apiserver_insecure_port: 8080 # (http) # Kube-proxy proxyMode configuration. # Can be ipvs, iptables -kube_proxy_mode: iptables +kube_proxy_mode: iptables + +## Encrypting Secret Data at Rest (experimental) +kube_encrypt_secret_data: false # DNS configuration. # Kubernetes cluster name, also will be used as DNS domain diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 index 79b9e17e7374f1c505fdf9e4d75aded32e53aafa..7e47e81b13cae7655ed70535f96293d2d196721f 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 @@ -7,4 +7,4 @@ metadata: labels: k8s-app: ingress-nginx data: - {{ ingress_nginx_configmap | to_nice_yaml }} + {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 index 5fb87594081606ee87f949fd7333004a9e0a69ce..0a87e91b7433c564e8730a9805aa920fe2666bf5 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 @@ -7,4 +7,4 @@ metadata: labels: k8s-app: ingress-nginx data: - {{ ingress_nginx_configmap_tcp_services | to_nice_yaml }} + {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 index bcb004bc9a23a81317b4515d2a33eb8cbaf45871..d943e57185cac2cdd1392df31624d0e2f388c619 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 @@ -7,4 +7,4 @@ metadata: labels: k8s-app: ingress-nginx data: - {{ ingress_nginx_configmap_udp_services | to_nice_yaml }} + {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }} diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 59e5288227518035a6d58166ffe0498e043ec763..a1b506d4e24225b2bfcbc90732131fcedd7d1141 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -92,3 +92,8 @@ kube_kubeadm_scheduler_extra_args: {} ## Variable for influencing kube-scheduler behaviour volume_cross_zone_attachment: false + +## Encrypting Secret Data at Rest +kube_encrypt_secret_data: false +kube_encrypt_token: "{{ lookup('password', inventory_dir + '/credentials/kube_encrypt_token length=32 chars=ascii_letters,digits') }}" +kube_encryption_algorithm: "aescbc" # Must be either: aescbc, secretbox or aesgcm diff --git a/roles/kubernetes/master/tasks/encrypt-at-rest.yml b/roles/kubernetes/master/tasks/encrypt-at-rest.yml new file mode 100644 index 0000000000000000000000000000000000000000..2e569b08bb63ca753f2c11ea232df494ba52ccc1 --- /dev/null +++ b/roles/kubernetes/master/tasks/encrypt-at-rest.yml @@ -0,0 +1,10 @@ +--- +- name: Write secrets for encrypting secret data at rest + template: + src: secrets_encryption.yaml.j2 + dest: "{{ kube_config_dir }}/ssl/secrets_encryption.yaml" + owner: root + group: "{{ kube_cert_group }}" + mode: 0640 + tags: + - kube-apiserver diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 04ad307fd536260bf0064c2c171fbb98cc6f2b67..daa10fd796ecd521f843d706791d879b547e3b53 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -12,6 +12,9 @@ - import_tasks: users-file.yml when: kube_basic_auth|default(true) +- import_tasks: encrypt-at-rest.yml + when: kube_encrypt_secret_data + - name: Compare host kubectl with hyperkube container command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl" register: kubectl_task_compare_result diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index ed1cc7add3669d3279aaed7972776a56223bf716..a15107c94d5c9e8e2531213c494d413e9bab7e4a 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -37,6 +37,7 @@ apiServerExtraArgs: admission-control: {{ kube_apiserver_admission_control | join(',') }} apiserver-count: "{{ kube_apiserver_count }}" service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" {% if kube_basic_auth|default(true) %} basic-auth-file: {{ kube_users_dir }}/known_users.csv {% endif %} @@ -52,6 +53,9 @@ apiServerExtraArgs: {% if kube_oidc_groups_claim is defined %} oidc-groups-claim: {{ kube_oidc_groups_claim }} {% endif %} +{% endif %} +{% if kube_encrypt_secret_data %} + experimental-encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml {% endif %} storage-backend: {{ kube_apiserver_storage_backend }} {% if kube_api_runtime_config is defined %} @@ -59,7 +63,7 @@ apiServerExtraArgs: {% endif %} allow-privileged: "true" {% for key in kube_kubeadm_apiserver_extra_args %} - {{ key }}: {{ kube_kubeadm_apiserver_extra_args[key] }} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" {% endfor %} controllerManagerExtraArgs: node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} @@ -69,12 +73,12 @@ controllerManagerExtraArgs: feature-gates: {{ kube_feature_gates|join(',') }} {% endif %} {% for key in kube_kubeadm_controller_extra_args %} - {{ key }}: {{ kube_kubeadm_controller_extra_args[key] }} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" {% endfor %} {% if kube_kubeadm_scheduler_extra_args|length > 0 %} schedulerExtraArgs: {% for key in kube_kubeadm_scheduler_extra_args %} - {{ key }}: {{ kube_kubeadm_scheduler_extra_args[key] }} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" {% endfor %} {% endif %} apiServerCertSANs: diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 0dbe93caba3d0f83b83cf95bb0bc4f9f75239844..c1685410dca4fec4473b7c7642eb2b91b8714662 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -103,6 +103,9 @@ spec: {% if authorization_modes %} - --authorization-mode={{ authorization_modes|join(',') }} {% endif %} +{% if kube_encrypt_secret_data %} + - --experimental-encryption-provider-config={{ kube_config_dir }}/ssl/secrets_encryption.yaml +{% endif %} {% if kube_feature_gates %} - --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} diff --git a/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 b/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..84c6a4ea808e83dd2bc8ee8750d9541dae8ec567 --- /dev/null +++ b/roles/kubernetes/master/templates/secrets_encryption.yaml.j2 @@ -0,0 +1,11 @@ +kind: EncryptionConfig +apiVersion: v1 +resources: + - resources: + - secrets + providers: + - {{ kube_encryption_algorithm }}: + keys: + - name: key + secret: {{ kube_encrypt_token | b64encode }} + - identity: {} diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index f67c72bf8d4541f02f0645e62a8c1489d0a3c07b..5be20d533e57aa11709aa885b130fb3d7c6bb14d 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -1,4 +1,4 @@ -### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/10-kubeadm.conf +### Upstream source https://github.com/kubernetes/release/blob/master/debian/xenial/kubeadm/channel/stable/etc/systemd/system/kubelet.service.d/ ### All upstream values should be present in this file # logging to stderr means we get it in the systemd journal @@ -23,13 +23,14 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if kubelet_authentication_token_webhook %} --authentication-token-webhook \ {% endif %} +{% if kubelet_authorization_mode_webhook %} --authorization-mode=Webhook \ +{% endif %} --client-ca-file={{ kube_cert_dir }}/ca.crt \ --pod-manifest-path={{ kube_manifest_dir }} \ --cadvisor-port={{ kube_cadvisor_port }} \ {# end kubeadm specific settings #} --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ ---kube-reserved cpu={{ kube_cpu_reserved }},memory={{ kube_memory_reserved|regex_replace('Mi', 'M') }} \ --node-status-update-frequency={{ kubelet_status_update_frequency }} \ --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml index 0e4346f674dbad30ad895bd5dc925013da840306..8ac8a901b62218affc5ebc6891b89c38979661f2 100644 --- a/tests/files/gce_centos7-flannel-addons.yml +++ b/tests/files/gce_centos7-flannel-addons.yml @@ -15,3 +15,4 @@ etcd_deployment_type: host deploy_netchecker: true kubedns_min_replicas: 1 cloud_provider: gce +kube_encrypt_secret_data: true