diff --git a/docs/vars.md b/docs/vars.md index 1c36e67344acd1de664142a4fbe3ae864ed38741..0e83b18311a7034dc5056e373fb0ec31720603b4 100644 --- a/docs/vars.md +++ b/docs/vars.md @@ -281,6 +281,11 @@ node_taints: * `audit_webhook_batch_max_wait`: 1s * *kubectl_alias* - Bash alias of kubectl to interact with Kubernetes cluster much easier. +* *remove_anonymous_access* - When set to `true`, removes the `kubeadm:bootstrap-signer-clusterinfo` rolebinding created by kubeadm. + By default, kubeadm creates a rolebinding in the `kube-public` namespace which grants permissions to anonymous users. This rolebinding allows kubeadm to discover and validate cluster information during the join phase. + In a nutshell, this option removes the rolebinding after the init phase of the first control plane node and then configures kubeadm to use file discovery for the join phase of other nodes. + This option does not remove the anonymous authentication feature of the API server. + ### Custom flags for Kube Components For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments. diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml index d1a640fe494e3f5b707b104fe40da0b9fde12acd..c25d495bee276dc33ca90838f29804a55586205a 100644 --- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml @@ -371,3 +371,6 @@ kubeadm_patches: enabled: false source_dir: "{{ inventory_dir }}/patches" dest_dir: "{{ kube_config_dir }}/patches" + +# Set to true to remove the role binding to anonymous users created by kubeadm +remove_anonymous_access: false diff --git a/roles/kubernetes/control-plane/defaults/main/main.yml b/roles/kubernetes/control-plane/defaults/main/main.yml index fd7047767cd269844846f4af886e37e4a641d058..df92c419be745732e580a2639b4f6a721e5c8ffc 100644 --- a/roles/kubernetes/control-plane/defaults/main/main.yml +++ b/roles/kubernetes/control-plane/defaults/main/main.yml @@ -240,3 +240,6 @@ kubeadm_upgrade_auto_cert_renewal: true kube_apiserver_tracing: false kube_apiserver_tracing_endpoint: 0.0.0.0:4317 kube_apiserver_tracing_sampling_rate_per_million: 100 + +# Enable kubeadm file discovery if anonymous access has been removed +kubeadm_use_file_discovery: "{{ remove_anonymous_access }}" diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml index f3fd207c44fc7d8d60c59bde06ec5de24bd5b5bb..e10ef7fabae0f745a822c3686bc78e7d02fbbde7 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml @@ -63,6 +63,26 @@ - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists - not kube_external_ca_mode +- name: Get kubeconfig for join discovery process + command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'" + register: kubeconfig_file_discovery + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + when: + - kubeadm_use_file_discovery + - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists + +- name: Copy discovery kubeconfig + copy: + dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" + content: "{{ kubeconfig_file_discovery.stdout }}" + owner: "root" + mode: 0644 + when: + - inventory_hostname != first_kube_control_plane + - kubeadm_use_file_discovery + - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists + - name: Joining control plane node to the cluster. command: >- {{ bin_dir }}/kubeadm join diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml index 1f4ff20a3c64cdd804e0ad4ab9a0e3d525576d10..ceaafa06c55d81ee1fe7dcccb86a9b41c2e71d19 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-setup.yml @@ -221,12 +221,16 @@ {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }} changed_when: false when: - - inventory_hostname == first_kube_control_plane + - inventory_hostname == first_kube_control_plane - kubeadm_token is defined - kubeadm_refresh_token tags: - kubeadm_token +- name: Remove binding to anonymous user + command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found" + when: inventory_hostname == first_kube_control_plane and remove_anonymous_access + - name: Create kubeadm token for joining nodes with 24h expiration (default) command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create" changed_when: false diff --git a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml index 12ab0b934694e5d3b9791cabdf1289247be871b4..7638a89686437f0141e63f881f72527cc2df3069 100644 --- a/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml +++ b/roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml @@ -53,6 +53,10 @@ PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}" notify: Master | restart kubelet +- name: Kubeadm | Remove binding to anonymous user + command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found" + when: remove_anonymous_access + - name: Kubeadm | clean kubectl cache to refresh api types file: path: "{{ item }}" diff --git a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 index c950d00b391fd3ca541dff9ddb7e53593b1fbb3f..cd19b5c2e5f1fb620fbd49c4befb7d9d54e0bb6b 100644 --- a/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 +++ b/roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2 @@ -1,6 +1,10 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: JoinConfiguration discovery: +{% if kubeadm_use_file_discovery %} + file: + kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml +{% else %} bootstrapToken: {% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} @@ -9,6 +13,7 @@ discovery: {% endif %} token: {{ kubeadm_token }} unsafeSkipCAVerification: true +{% endif %} timeout: {{ discovery_timeout }} tlsBootstrapToken: {{ kubeadm_token }} controlPlane: diff --git a/roles/kubernetes/kubeadm/defaults/main.yml b/roles/kubernetes/kubeadm/defaults/main.yml index 61b132e61f648b92cf514e3c825f16beefd90d4f..5047de5094a3afbb29132963cc6c966e98fd5383 100644 --- a/roles/kubernetes/kubeadm/defaults/main.yml +++ b/roles/kubernetes/kubeadm/defaults/main.yml @@ -4,6 +4,9 @@ discovery_timeout: 60s kubeadm_join_timeout: 120s +# Enable kubeadm file discovery if anonymous access has been removed +kubeadm_use_file_discovery: "{{ remove_anonymous_access }}" + # If non-empty, will use this string as identification instead of the actual hostname kube_override_hostname: >- {%- if cloud_provider is defined and cloud_provider in ['aws'] -%} diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 4a65dbbc9da9b40874e5fe1f0b2d1f8fb5368843..e8b5dceb61b1a6563d78d9b4af8e5eb52aeabe12 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -57,6 +57,24 @@ set_fact: kubeadmConfig_api_version: v1beta3 +- name: Get kubeconfig for join discovery process + command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'" + register: kubeconfig_file_discovery + run_once: true + delegate_to: "{{ groups['kube_control_plane'] | first }}" + when: kubeadm_use_file_discovery + +- name: Copy discovery kubeconfig + copy: + dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml" + content: "{{ kubeconfig_file_discovery.stdout }}" + owner: "root" + mode: 0644 + when: + - not is_kube_master + - not kubelet_conf.stat.exists + - kubeadm_use_file_discovery + - name: Create kubeadm client config template: src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2" diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 index 5104ecfb949b5e98a19915f7e45b9533a24b618e..3b3bc57de4688c4fe9e753a0d78be737df0f1009 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2 @@ -2,6 +2,10 @@ apiVersion: kubeadm.k8s.io/v1beta3 kind: JoinConfiguration discovery: +{% if kubeadm_use_file_discovery %} + file: + kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml +{% else %} bootstrapToken: {% if kubeadm_config_api_fqdn is defined %} apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} @@ -14,6 +18,7 @@ discovery: - sha256:{{ kubeadm_ca_hash.stdout }} {% else %} unsafeSkipCAVerification: true +{% endif %} {% endif %} timeout: {{ discovery_timeout }} tlsBootstrapToken: {{ kubeadm_token }} diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index d2dbccc22b9818c4778ac27e919aa4b175849eec..ed71d8a066feb06834a0dc5e2dc51e4d5a7010a7 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -6,6 +6,8 @@ ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='s # selinux state preinstall_selinux_state: permissive +# Setting this value to false will fail +# For details, read this comment https://github.com/kubernetes-sigs/kubespray/pull/11016#issuecomment-2004985001 kube_api_anonymous_auth: true # Default value, but will be set to true automatically if detected @@ -50,6 +52,9 @@ kubeadm_join_phases_skip_default: [] kubeadm_join_phases_skip: >- {{ kubeadm_join_phases_skip_default }} +# Set to true to remove the role binding to anonymous users created by kubeadm +remove_anonymous_access: false + # A string slice of values which specify the addresses to use for NodePorts. # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). # The default empty string slice ([]) means to use all local addresses. diff --git a/tests/files/packet_debian11-calico-upgrade.yml b/tests/files/packet_debian11-calico-upgrade.yml index 1b05714e41121cb0b3099e229079e97251102e42..94aba7b9247bd5ae30e2bfabd923cef304702136 100644 --- a/tests/files/packet_debian11-calico-upgrade.yml +++ b/tests/files/packet_debian11-calico-upgrade.yml @@ -11,3 +11,6 @@ calico_network_backend: bird # Needed to bypass deprecation check ignore_assert_errors: true + +# Remove anonymous access to cluster +remove_anonymous_access: true diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml index 55cbd506374eb9057b3ef5f36801b6cd7f19a24f..c494810cf0719ee8dc090d8f2c37ad1ced3e8da6 100644 --- a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml +++ b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml @@ -104,3 +104,6 @@ kube_cert_group: root # kube-system namespace is exempted by default kube_pod_security_use_default: true kube_pod_security_default_enforce: restricted + +# Remove anonymous access to cluster +remove_anonymous_access: true diff --git a/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml b/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml index 99f736544fdb090897c5d6fef85818e9f4eb172e..ba9d7b34b8d0765fe61193bc40c377798057aba2 100644 --- a/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml +++ b/tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml @@ -9,3 +9,6 @@ etcd_deployment_type: kubeadm # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko kube_proxy_mode: iptables enable_nodelocaldns: False + +# Remove anonymous access to cluster +remove_anonymous_access: true