diff --git a/roles/kubernetes/client/tasks/main.yml b/roles/kubernetes/client/tasks/main.yml index a572ba69057d1e3db3f76b14a676b1fe920da5fe..7bef942738d0def583beeb37f2c774345d79c19a 100644 --- a/roles/kubernetes/client/tasks/main.yml +++ b/roles/kubernetes/client/tasks/main.yml @@ -62,7 +62,7 @@ --client-name kubernetes-admin --org system:masters {% endif %} - --cert-dir {{ kube_config_dir }}/ssl + --cert-dir {{ kube_cert_dir }} --apiserver-advertise-address {{ external_apiserver_address }} --apiserver-bind-port {{ external_apiserver_port }} {% if kubeadm_version is version('v1.14.0', '>=') %} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 index 78e861aeb9b0ded6a11afb75d80ee60e591dcc83..1f0f9d8478e2a4867de2999e4c097c2adea32d7e 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 @@ -1,3 +1,4 @@ +--- apiVersion: kubeadm.k8s.io/v1beta1 kind: JoinConfiguration discovery: @@ -8,7 +9,8 @@ discovery: apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}} {% endif %} token: {{ kubeadm_token }} - unsafeSkipCAVerification: true + caCertHashes: + - sha256:{{ kubeadm_ca_hash.stdout }} timeout: {{ discovery_timeout }} tlsBootstrapToken: {{ kubeadm_token }} caCertPath: {{ kube_cert_dir }}/ca.crt diff --git a/roles/kubernetes/node/tasks/azure-credential-check.yml b/roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml similarity index 100% rename from roles/kubernetes/node/tasks/azure-credential-check.yml rename to roles/kubernetes/node/tasks/cloud-credentials/azure-credential-check.yml diff --git a/roles/kubernetes/node/tasks/openstack-credential-check.yml b/roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml similarity index 100% rename from roles/kubernetes/node/tasks/openstack-credential-check.yml rename to roles/kubernetes/node/tasks/cloud-credentials/openstack-credential-check.yml diff --git a/roles/kubernetes/node/tasks/vsphere-credential-check.yml b/roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml similarity index 100% rename from roles/kubernetes/node/tasks/vsphere-credential-check.yml rename to roles/kubernetes/node/tasks/cloud-credentials/vsphere-credential-check.yml diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 52ad10b6a31ef310e0b91d3efa11a5cae5a19ee9..fb38da86273a27f23821c6a9d2f9c382b4fc7ce5 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -73,10 +73,3 @@ args: creates: "{{ bin_dir }}/socat" when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] - -- name: install | Write kubelet systemd init file - template: - src: "kubelet.host.service.j2" - dest: "/etc/systemd/system/kubelet.service" - backup: "yes" - notify: restart kubelet diff --git a/roles/kubernetes/node/tasks/kubelet.yml b/roles/kubernetes/node/tasks/kubelet.yml new file mode 100644 index 0000000000000000000000000000000000000000..6963c965dd41009d283a67cc6c5bf0040506ad95 --- /dev/null +++ b/roles/kubernetes/node/tasks/kubelet.yml @@ -0,0 +1,71 @@ +--- +- name: Make sure dynamic kubelet configuration directory is writeable + file: + path: "{{ dynamic_kubelet_configuration_dir }}" + mode: 0600 + state: directory + when: dynamic_kubelet_configuration + +- name: gets the kubeadm version + command: "{{ bin_dir }}/kubeadm version -o short" + register: kubeadm_output + +- name: sets kubeadm api version to v1beta1 + set_fact: + kubeadmConfig_api_version: v1beta1 + when: kubeadm_output.stdout is version('v1.13.0', '>=') + tags: + - kubelet + - kubeadm + +- name: Write kubelet environment config file (kubeadm) + template: + src: "kubelet.env.{{ kubeadmConfig_api_version }}.j2" + dest: "{{ kube_config_dir }}/kubelet.env" + backup: yes + notify: restart kubelet + when: kubeadm_output.stdout is version('v1.13.0', '>=') + tags: + - kubelet + - kubeadm + +- name: Write kubelet environment config file (kubeadm) + template: + src: "kubelet.env.j2" + dest: "{{ kube_config_dir }}/kubelet.env" + backup: yes + notify: restart kubelet + when: kubeadm_output.stdout is version('v1.13.0', '<') + tags: + - kubelet + - kubeadm + +- name: Write kubelet config file + template: + src: "kubelet-config.{{ kubeadmConfig_api_version }}.yaml.j2" + dest: "{{ kube_config_dir }}/kubelet-config.yaml" + when: kubeadm_output.stdout is version('v1.13.0', '>=') + tags: + - kubelet + - kubeadm + +- name: Write kubelet systemd init file + template: + src: "kubelet.service.j2" + dest: "/etc/systemd/system/kubelet.service" + backup: "yes" + notify: restart kubelet + tags: + - kubelet + - kubeadm + +- name: flush_handlers and reload-systemd + meta: flush_handlers + +- name: Enable kubelet + service: + name: kubelet + enabled: yes + state: started + tags: + - kubelet \ No newline at end of file diff --git a/roles/kubernetes/node/tasks/haproxy.yml b/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml similarity index 93% rename from roles/kubernetes/node/tasks/haproxy.yml rename to roles/kubernetes/node/tasks/loadbalancer/haproxy.yml index ed899a3f4b0da0d0f640805d148e254a0749c897..e7cb579a6021b6a80535e37a48675f6797f2a2bd 100644 --- a/roles/kubernetes/node/tasks/haproxy.yml +++ b/roles/kubernetes/node/tasks/loadbalancer/haproxy.yml @@ -18,7 +18,7 @@ - name: haproxy | Write haproxy configuration template: - src: haproxy.cfg.j2 + src: "loadbalancer/haproxy.cfg.j2" dest: "{{ haproxy_config_dir }}/haproxy.cfg" owner: root mode: 0755 diff --git a/roles/kubernetes/node/tasks/nginx-proxy.yml b/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml similarity index 93% rename from roles/kubernetes/node/tasks/nginx-proxy.yml rename to roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml index 36c1d7306b604c2fa6ba87678e1e95f62bf19b1c..e1910137952253d85dc9c6d94599f8df716e8263 100644 --- a/roles/kubernetes/node/tasks/nginx-proxy.yml +++ b/roles/kubernetes/node/tasks/loadbalancer/nginx-proxy.yml @@ -18,7 +18,7 @@ - name: nginx-proxy | Write nginx-proxy configuration template: - src: nginx.conf.j2 + src: "loadbalancer/nginx.conf.j2" dest: "{{ nginx_config_dir }}/nginx.conf" owner: root mode: 0755 diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 0a593b3a0f33df5f159f81ac80005e41047350f8..71a2679d002529902956399c62240ee03e6be3d1 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -17,7 +17,7 @@ tags: - kubelet -- import_tasks: nginx-proxy.yml +- import_tasks: loadbalancer/nginx-proxy.yml when: - not is_kube_master - loadbalancer_apiserver_localhost @@ -25,7 +25,7 @@ tags: - nginx -- import_tasks: haproxy.yml +- import_tasks: loadbalancer/haproxy.yml when: - not is_kube_master - loadbalancer_apiserver_localhost @@ -33,23 +33,6 @@ tags: - haproxy -- name: Make sure dynamic kubelet configuration directory is writeable - file: - path: "{{ dynamic_kubelet_configuration_dir }}" - mode: 0600 - state: directory - when: dynamic_kubelet_configuration - -- name: Write kubelet config file (kubeadm) - template: - src: kubelet.kubeadm.env.j2 - dest: "{{ kube_config_dir }}/kubelet.env" - backup: yes - notify: restart kubelet - tags: - - kubelet - - kubeadm - - name: Ensure nodePort range is reserved sysctl: name: net.ipv4.ip_local_reserved_ports @@ -99,7 +82,7 @@ name: "{{ item }}" state: present sysctl_file: "{{ sysctl_file_path }}" - value: 1 + value: "1" reload: yes when: sysctl_bridge_nf_call_iptables.rc == 0 with_items: @@ -175,7 +158,7 @@ tags: - kube-proxy -- include_tasks: "{{ cloud_provider }}-credential-check.yml" +- include_tasks: "cloud-credentials/{{ cloud_provider }}-credential-check.yml" when: - cloud_provider is defined - cloud_provider in [ 'openstack', 'azure', 'vsphere' ] @@ -200,7 +183,7 @@ - name: Write cloud-config template: - src: "{{ cloud_provider }}-cloud-config.j2" + src: "cloud-configs/{{ cloud_provider }}-cloud-config.j2" dest: "{{ kube_config_dir }}/cloud_config" group: "{{ kube_cert_group }}" mode: 0640 @@ -211,13 +194,7 @@ tags: - cloud-provider -# reload-systemd -- meta: flush_handlers - -- name: Enable kubelet - service: - name: kubelet - enabled: yes - state: started +- import_tasks: kubelet.yml tags: - kubelet + - kubeadm \ No newline at end of file diff --git a/roles/kubernetes/node/templates/aws-cloud-config.j2 b/roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2 similarity index 100% rename from roles/kubernetes/node/templates/aws-cloud-config.j2 rename to roles/kubernetes/node/templates/cloud-configs/aws-cloud-config.j2 diff --git a/roles/kubernetes/node/templates/azure-cloud-config.j2 b/roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 similarity index 100% rename from roles/kubernetes/node/templates/azure-cloud-config.j2 rename to roles/kubernetes/node/templates/cloud-configs/azure-cloud-config.j2 diff --git a/roles/kubernetes/node/templates/openstack-cloud-config.j2 b/roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2 similarity index 100% rename from roles/kubernetes/node/templates/openstack-cloud-config.j2 rename to roles/kubernetes/node/templates/cloud-configs/openstack-cloud-config.j2 diff --git a/roles/kubernetes/node/templates/vsphere-cloud-config.j2 b/roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2 similarity index 100% rename from roles/kubernetes/node/templates/vsphere-cloud-config.j2 rename to roles/kubernetes/node/templates/cloud-configs/vsphere-cloud-config.j2 diff --git a/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..2beef8bf157ec4c1ff0144968198e2e5c5019cae --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet-config.v1beta1.yaml.j2 @@ -0,0 +1,57 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +nodeStatusUpdateFrequency: "{{ kubelet_status_update_frequency }}" +failSwapOn: {{ kubelet_fail_swap_on|default(true) }} +authentication: + anonymous: + enabled: false + webhook: + enabled: {{ kubelet_authentication_token_webhook }} + x509: + clientCAFile: {{ kube_cert_dir }}/ca.crt +authorization: +{% if kubelet_authorization_mode_webhook %} + mode: Webhook +{% else %} + mode: AlwaysAllow +{% endif %} +{% if kubelet_enforce_node_allocatable is defined and kubelet_enforce_node_allocatable != "\"\"" %} +{% set kubelet_enforce_node_allocatable_list = kubelet_enforce_node_allocatable.split() %} +enforceNodeAllocatable: +{% for item in kubelet_enforce_node_allocatable_list %} +- {{ item }} +{% endfor %} +{% endif %} +staticPodPath: {{ kube_manifest_dir }} +cgroupDriver: {{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} +maxPods: {{ kubelet_max_pods }} +address: {{ kubelet_bind_address }} +readOnlyPort: {{ kube_read_only_port }} +kubeletCgroups: {{ kubelet_kubelet_cgroups }} +clusterDomain: {{ dns_domain }} +{# DNS settings for kubelet #} +{% if enable_nodelocaldns %} +{% set kubelet_cluster_dns = [nodelocaldns_ip] %} +{% elif dns_mode in ['coredns'] %} +{% set kubelet_cluster_dns = [skydns_server] %} +{% elif dns_mode == 'coredns_dual' %} +{% set kubelet_cluster_dns = [skydns_server,skydns_server_secondary] %} +{% elif dns_mode == 'manual' %} +{% set kubelet_cluster_dns = [manual_dns_server] %} +{% else %} +{% set kubelet_cluster_dns = [] %} +{% endif %} +clusterDNS: +{% for dns_address in kubelet_cluster_dns %} +- {{ dns_address }} +{% endfor %} +{# Node reserved CPU/memory #} +kubeReserved: +{% if is_kube_master|bool %} + cpu: {{ kube_master_cpu_reserved }} + memory: {{ kube_master_memory_reserved|regex_replace('Mi', 'M') }} +{% else %} + cpu: {{ kube_cpu_reserved }} + memory: {{ kube_memory_reserved|regex_replace('Mi', 'M') }} +{% endif %} +resolvConf: "{{ kube_resolv_conf }}" \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.env.j2 similarity index 100% rename from roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 rename to roles/kubernetes/node/templates/kubelet.env.j2 diff --git a/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 new file mode 100644 index 0000000000000000000000000000000000000000..e86ee456520f4c876d78a163d0b184f06de26913 --- /dev/null +++ b/roles/kubernetes/node/templates/kubelet.env.v1beta1.j2 @@ -0,0 +1,78 @@ +KUBE_LOGTOSTDERR="--logtostderr=true" +KUBE_LOG_LEVEL="--v={{ kube_log_level }}" +KUBELET_ADDRESS="--node-ip={{ kubelet_address }}" +{% if kube_override_hostname|default('') %} +KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" +{% endif %} + +{# Base kubelet args #} +{% set kubelet_args_base -%} +{# start kubeadm specific settings #} +--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ +--config={{ kube_config_dir }}/kubelet-config.yaml \ +--kubeconfig={{ kube_config_dir }}/kubelet.conf \ +{% if kube_version is version('v1.8', '<') %} +--require-kubeconfig \ +{% endif %} +{% if kube_version is version('v1.12.0', '<') %} +--cadvisor-port={{ kube_cadvisor_port }} \ +{% endif %} +{# end kubeadm specific settings #} +--pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ +{% if container_manager == 'docker' and kube_version is version('v1.12.0', '<') %} +--docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ +{% endif %} +{% if container_manager == 'crio' %} +--container-runtime=remote \ +--container-runtime-endpoint=/var/run/crio/crio.sock \ +{% endif %} +{% if kube_version is version('v1.8', '<') %} +--experimental-fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ +{% endif %} +{% if dynamic_kubelet_configuration %} +--dynamic-config-dir={{ dynamic_kubelet_configuration_dir }} \ +{% endif %} +--runtime-cgroups={{ kubelet_runtime_cgroups }} \ +{% endset %} + +{# Kubelet node labels #} +{% set role_node_labels = [] %} +{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} +{% if inventory_hostname in nvidia_gpu_nodes %} +{% set dummy = role_node_labels.append('nvidia.com/gpu=true') %} +{% endif %} +{% endif %} + +{% set inventory_node_labels = [] %} +{% if node_labels is defined and node_labels is mapping %} +{% for labelname, labelvalue in node_labels.items() %} +{% set dummy = inventory_node_labels.append('%s=%s'|format(labelname, labelvalue)) %} +{% endfor %} +{% endif %} +{% set all_node_labels = role_node_labels + inventory_node_labels %} + +{# Kubelet node taints for gpu #} +{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %} +{% if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %} +{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} +{% elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %} +{% set node_taints = [] %} +{% set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %} +{% endif %} +{% endif %} + +KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %}--node-labels={{ all_node_labels | join(',') }} {% if kube_feature_gates %} --feature-gates={{ kube_feature_gates|join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube-node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}" +{% if kube_network_plugin is defined and kube_network_plugin in ["calico", "canal", "cni", "flannel", "weave", "contiv", "cilium", "kube-router"] %} +KUBELET_NETWORK_PLUGIN="--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin" +{% elif kube_network_plugin is defined and kube_network_plugin == "cloud" %} +KUBELET_NETWORK_PLUGIN="--hairpin-mode=promiscuous-bridge --network-plugin=kubenet" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} +KUBELET_CLOUDPROVIDER="--cloud-provider={{ cloud_provider }} --cloud-config={{ kube_config_dir }}/cloud_config" +{% elif cloud_provider is defined and cloud_provider in ["external"] %} +KUBELET_CLOUDPROVIDER="--cloud-provider=external --cloud-config={{ kube_config_dir }}/cloud_config" +{% else %} +KUBELET_CLOUDPROVIDER="" +{% endif %} + +PATH={{ bin_dir }}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ No newline at end of file diff --git a/roles/kubernetes/node/templates/kubelet.host.service.j2 b/roles/kubernetes/node/templates/kubelet.service.j2 similarity index 96% rename from roles/kubernetes/node/templates/kubelet.host.service.j2 rename to roles/kubernetes/node/templates/kubelet.service.j2 index 96de60d6e29c6d41c9f3998906e6d94ce467bbb3..4b4dce7c7379b4f646139007bda9e7f7f981aa81 100644 --- a/roles/kubernetes/node/templates/kubelet.host.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.service.j2 @@ -15,7 +15,6 @@ ExecStart={{ bin_dir }}/kubelet \ $KUBELET_ADDRESS \ $KUBELET_PORT \ $KUBELET_HOSTNAME \ - $KUBE_ALLOW_PRIV \ $KUBELET_ARGS \ $DOCKER_SOCKET \ $KUBELET_NETWORK_PLUGIN \ diff --git a/roles/kubernetes/node/templates/haproxy.cfg.j2 b/roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 similarity index 100% rename from roles/kubernetes/node/templates/haproxy.cfg.j2 rename to roles/kubernetes/node/templates/loadbalancer/haproxy.cfg.j2 diff --git a/roles/kubernetes/node/templates/nginx.conf.j2 b/roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 similarity index 100% rename from roles/kubernetes/node/templates/nginx.conf.j2 rename to roles/kubernetes/node/templates/loadbalancer/nginx.conf.j2 diff --git a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 index 0951f234218d25c3a5eb66a5c887260cb59e17be..4b8af602059d6bb968a4ce553b0d4a3b9c536954 100644 --- a/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 +++ b/roles/kubernetes/node/templates/node-kubeconfig.yaml.j2 @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Config clusters: diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index c0401cd2419498bb1f294e6d5e1593ea8ec71d68..8a9478380949923adc6f4ba4b42c2f599877ec21 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -314,7 +314,7 @@ rbac_enabled: "{{ 'RBAC' in authorization_modes }}" kubelet_authentication_token_webhook: true # When enabled, access to the kubelet API requires authorization by delegation to the API server -kubelet_authorization_mode_webhook: false +kubelet_authorization_mode_webhook: true # kubelet uses certificates for authenticating to the Kubernetes API # Automatically generate a new key and request a new certificate from the Kubernetes API as the current certificate approaches expiration