diff --git a/README.md b/README.md index 4fe0520d1d881effcf6a3aae217196f61b73be77..cb2bdbef403dcc1433a81ba6195b213cb752d84b 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ Note: Upstart/SysV init based OS types are not supported. - Core - [kubernetes](https://github.com/kubernetes/kubernetes) v1.18.5 - - [etcd](https://github.com/coreos/etcd) v3.3.12 + - [etcd](https://github.com/coreos/etcd) v3.4.3 - [docker](https://www.docker.com/) v19.03 (see note) - [containerd](https://containerd.io/) v1.2.13 - [cri-o](http://cri-o.io/) v1.17 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS) diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 8b1159ba6e19e034542ce3714321e07211e47cb6..5e152f27c5bcb95b870e1596739bf598a3b3fd48 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -51,7 +51,7 @@ image_arch: "{{host_architecture | default('amd64')}}" # Versions kube_version: v1.18.5 kubeadm_version: "{{ kube_version }}" -etcd_version: v3.3.12 +etcd_version: v3.4.3 # gcr and kubernetes image repo define gcr_image_repo: "gcr.io" @@ -376,8 +376,8 @@ etcd_binary_checksums: # Etcd does not have arm32 builds at the moment, having some dummy value is # required to avoid "no attribute" error arm: 0 - arm64: 170b848ac1a071fe7d495d404a868a2c0090750b2944f8a260ef1c6125b2b4f4 - amd64: dc5d82df095dae0a2970e4d870b6929590689dd707ae3d33e7b86da0f7f211b6 + arm64: 01bd849ad99693600bd59db8d0e66ac64aac1e3801900665c31bd393972e3554 + amd64: 6c642b723a86941b99753dff6c00b26d3b033209b15ee33325dc8e7f4cd68f07 cni_binary_checksums: arm: 28e61b5847265135dc1ca397bf94322ecce4acab5c79cc7d360ca3f6a655bdb7 arm64: 43fbf750c5eccb10accffeeb092693c32b236fb25d919cf058c91a677822c999 diff --git a/roles/etcd/handlers/backup.yml b/roles/etcd/handlers/backup.yml index 32c0a3d401b50f9a71e92cc79ab84ccf80bfc123..966da538b55c8b7e75211996f59283418aa24d84 100644 --- a/roles/etcd/handlers/backup.yml +++ b/roles/etcd/handlers/backup.yml @@ -49,7 +49,7 @@ snapshot save {{ etcd_backup_directory }}/snapshot.db environment: ETCDCTL_API: 3 - ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}" ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" diff --git a/roles/etcd/tasks/configure.yml b/roles/etcd/tasks/configure.yml index 4dcf7c6b96c06888410e2869e53bd4316435b67d..56d5f86c8b9c222f29a8709533edc11bf2d99ae9 100644 --- a/roles/etcd/tasks/configure.yml +++ b/roles/etcd/tasks/configure.yml @@ -1,6 +1,6 @@ --- - name: Configure | Check if etcd cluster is healthy - shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'" register: etcd_cluster_is_healthy failed_when: false changed_when: false @@ -10,14 +10,14 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Check if etcd-events cluster is healthy - shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'" register: etcd_events_cluster_is_healthy failed_when: false changed_when: false @@ -27,11 +27,11 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - include_tasks: refresh_config.yml when: is_etcd_master @@ -74,12 +74,11 @@ when: is_etcd_master and etcd_events_cluster_setup - name: Configure | Wait for etcd cluster to be healthy - shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'" register: etcd_cluster_is_healthy until: etcd_cluster_is_healthy.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" - ignore_errors: false changed_when: false check_mode: no run_once: yes @@ -89,19 +88,18 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Wait for etcd-events cluster to be healthy - shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'" + shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'" register: etcd_events_cluster_is_healthy until: etcd_events_cluster_is_healthy.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" - ignore_errors: false changed_when: false check_mode: no run_once: yes @@ -111,14 +109,14 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Check if member is in etcd cluster - shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}" + shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}" register: etcd_member_in_cluster ignore_errors: true changed_when: false @@ -127,14 +125,14 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Check if member is in etcd-events cluster - shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}" + shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}" register: etcd_events_member_in_cluster ignore_errors: true changed_when: false @@ -143,11 +141,11 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Join member(s) to etcd cluster one at a time include_tasks: join_etcd_member.yml diff --git a/roles/etcd/tasks/join_etcd-events_member.yml b/roles/etcd/tasks/join_etcd-events_member.yml index e16811702dc93e1af3a7124ac87f2aa5f4141ea0..a6a197a7422f9323835380cbd917c0bb4e89c2a4 100644 --- a/roles/etcd/tasks/join_etcd-events_member.yml +++ b/roles/etcd/tasks/join_etcd-events_member.yml @@ -1,15 +1,16 @@ --- - name: Join Member | Add member to etcd-events cluster - shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_events_peer_url }}" + shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}" register: member_add_result until: member_add_result.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - include_tasks: refresh_config.yml vars: @@ -24,17 +25,18 @@ {%- endfor -%} - name: Join Member | Ensure member is in etcd-events cluster - shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_events_access_address }}" + shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_events_access_address }}" register: etcd_events_member_in_cluster changed_when: false check_mode: no tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - name: Configure | Ensure etcd-events is running service: diff --git a/roles/etcd/tasks/join_etcd_member.yml b/roles/etcd/tasks/join_etcd_member.yml index bea484c3747e42d7614b0e2667c61760791c8272..e7ee2a348f621c94466393890406a0610980024c 100644 --- a/roles/etcd/tasks/join_etcd_member.yml +++ b/roles/etcd/tasks/join_etcd_member.yml @@ -1,16 +1,16 @@ --- - name: Join Member | Add member to etcd cluster - shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_peer_url }}" + shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}" register: member_add_result until: member_add_result.rc == 0 retries: "{{ etcd_retries }}" delay: "{{ retry_stagger | random + 3 }}" environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - include_tasks: refresh_config.yml vars: @@ -25,18 +25,18 @@ {%- endfor -%} - name: Join Member | Ensure member is in etcd cluster - shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}" + shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}" register: etcd_member_in_cluster changed_when: false check_mode: no tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" - name: Configure | Ensure etcd is running service: diff --git a/roles/etcd/templates/etcd-events.env.j2 b/roles/etcd/templates/etcd-events.env.j2 index 3054b549c50fa27ada9e01e63e0b3923335447d3..99820fb2a675c32dd0c81b472429ea56ea7e42a6 100644 --- a/roles/etcd/templates/etcd-events.env.j2 +++ b/roles/etcd/templates/etcd-events.env.j2 @@ -1,7 +1,7 @@ ETCD_DATA_DIR={{ etcd_events_data_dir }} ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }} ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }} -ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %} +ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %} ETCD_METRICS={{ etcd_metrics }} ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2381,https://127.0.0.1:2381 diff --git a/roles/etcd/templates/etcd.env.j2 b/roles/etcd/templates/etcd.env.j2 index 536fd681578b97e51a4ae2d6e1b1b3b161a242c9..f3114afd0ab825fcd2b4eacade9c7d0ecf513a65 100644 --- a/roles/etcd/templates/etcd.env.j2 +++ b/roles/etcd/templates/etcd.env.j2 @@ -2,7 +2,7 @@ ETCD_DATA_DIR={{ etcd_data_dir }} ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }} ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }} -ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %} +ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %} ETCD_METRICS={{ etcd_metrics }} {% if etcd_metrics_port is defined %} @@ -26,6 +26,8 @@ ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }} {% if etcd_log_package_levels is defined %} ETCD_LOG_PACKAGE_LEVELS={{ etcd_log_package_levels }} {% endif %} +# Flannel need etcd v2 API +ETCD_ENABLE_V2=true # TLS settings ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem @@ -48,6 +50,6 @@ ETCD_UNSUPPORTED_ARCH={{host_architecture}} # CLI settings ETCDCTL_ENDPOINTS=https://127.0.0.1:2379 -ETCDCTL_CA_FILE={{ etcd_cert_dir }}/ca.pem -ETCDCTL_KEY_FILE={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem -ETCDCTL_CERT_FILE={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem +ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem +ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem +ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index 921ef736ba2f1d56980d469d292cfcd9f9d4a8f4..5582ef10926811b343472c79e017998183b12bd4 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -26,10 +26,10 @@ - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"} - {s: "{{ kube_etcd_key_file }}", d: "key.pem"} +# Flannel need etcd v2 API - name: Canal | Set Flannel etcd configuration command: |- - {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \ - set /{{ cluster_name }}/network/config \ + {{ bin_dir }}/etcdctl set /{{ cluster_name }}/network/config \ '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }' register: output retries: 4 @@ -39,8 +39,11 @@ changed_when: false run_once: true environment: + ETCDCTL_API: 2 + ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}.pem" ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - name: Canal | Create canal node manifests template: diff --git a/roles/recover_control_plane/etcd/tasks/main.yml b/roles/recover_control_plane/etcd/tasks/main.yml index 25c94a9513d7387ed2971350be017913f53415a0..64cac81dad7182472e04fa756f58373cfd8d32b1 100644 --- a/roles/recover_control_plane/etcd/tasks/main.yml +++ b/roles/recover_control_plane/etcd/tasks/main.yml @@ -20,9 +20,10 @@ when: - groups['broken_etcd'] +# When there is an error, everything is printed in stderr_lines, even "is healthy" messages. - name: Set has_quorum fact set_fact: - has_quorum: "{{ etcd_endpoint_health.stdout_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}" + has_quorum: "{{ etcd_endpoint_health.stderr_lines | select('match', '.*is healthy.*') | list | length >= etcd_endpoint_health.stderr_lines | select('match', '.*is unhealthy.*') | list | length }}" - include_tasks: recover_lost_quorum.yml when: diff --git a/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml b/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml index fdd9d0b5fe62e94e53faab44c2a51dcaa30894d6..dc101180584ccbc37f9f4d5ad349eeae51961bf0 100644 --- a/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml +++ b/roles/recover_control_plane/etcd/tasks/recover_lost_quorum.yml @@ -1,7 +1,11 @@ --- - name: Save etcd snapshot - shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem snapshot save /tmp/snapshot.db" + shell: "{{ bin_dir }}/etcdctl snapshot save /tmp/snapshot.db" environment: + - ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + - ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + - ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + - ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}" - ETCDCTL_API: 3 when: etcd_snapshot is not defined @@ -22,8 +26,12 @@ state: absent - name: Restore etcd snapshot - shell: "{{ bin_dir }}/etcdctl --cacert {{ etcd_cert_dir }}/ca.pem --cert {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem --key {{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}" + shell: "{{ bin_dir }}/etcdctl snapshot restore /tmp/snapshot.db --name {{ etcd_member_name }} --initial-cluster {{ etcd_member_name }}={{ etcd_peer_url }} --initial-cluster-token k8s_etcd --initial-advertise-peer-urls {{ etcd_peer_url }} --data-dir {{ etcd_data_dir }}" environment: + - ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + - ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + - ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + - ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_API: 3 - name: Remove etcd snapshot diff --git a/roles/remove-node/post-remove/tasks/main.yml b/roles/remove-node/post-remove/tasks/main.yml index 6896ecba1e1d255cd2aa7f450798108b4d6b146e..37aac0df2398337a8e6fdd427fb966840bdf6fbe 100644 --- a/roles/remove-node/post-remove/tasks/main.yml +++ b/roles/remove-node/post-remove/tasks/main.yml @@ -2,4 +2,4 @@ - name: Delete node command: "{{ bin_dir }}/kubectl delete node {{ kube_override_hostname|default(inventory_hostname) }}" delegate_to: "{{ groups['kube-master']|first }}" - ignore_errors: yes + ignore_errors: yes \ No newline at end of file diff --git a/roles/remove-node/remove-etcd-node/tasks/main.yml b/roles/remove-node/remove-etcd-node/tasks/main.yml index ddde9ed27a0fcbd1b6a64516cb3a7bab5ba6c987..1b126e65815d5eb2b20ea64d802696ce7153bc14 100644 --- a/roles/remove-node/remove-etcd-node/tasks/main.yml +++ b/roles/remove-node/remove-etcd-node/tasks/main.yml @@ -18,7 +18,7 @@ - inventory_hostname in groups['etcd'] - name: Lookup etcd member id - shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep {{ node_ip }} | cut -d: -f1" + shell: "{{ bin_dir }}/etcdctl member list | grep {{ node_ip }} | cut -d: -f1" register: etcd_member_id ignore_errors: true changed_when: false @@ -26,32 +26,27 @@ tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" delegate_to: "{{ groups['etcd']|first }}" - when: - - inventory_hostname in groups['etcd'] + when: inventory_hostname in groups['etcd'] - name: Remove etcd member from cluster - shell: "{{ bin_dir }}/etcdctl --no-sync member remove {{ etcd_member_id.stdout }}" + shell: "{{ bin_dir }}/etcdctl member remove {{ etcd_member_id.stdout }}" register: etcd_member_in_cluster - ignore_errors: false - retries: 6 - delay: 5 - until: etcd_member_in_cluster.rc == 0 changed_when: false check_mode: no tags: - facts environment: - ETCDCTL_API: 2 + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ groups['etcd']|first }}-key.pem" - ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem" delegate_to: "{{ groups['etcd']|first }}" when: - inventory_hostname in groups['etcd'] diff --git a/scripts/collect-info.yaml b/scripts/collect-info.yaml index 4c203648e39dab49edca64602337b28943984e70..7524e349071c4f9c4aa85a6033bdc467d0000b9a 100644 --- a/scripts/collect-info.yaml +++ b/scripts/collect-info.yaml @@ -30,7 +30,7 @@ - name: errors_info cmd: journalctl -p err --no-pager - name: etcd_info - cmd: "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | default('http://127.0.0.1:2379') }} cluster-health" + cmd: "{{ bin_dir }}/etcdctl endpoint --cluster health" - name: calico_info cmd: "{{ bin_dir }}/calicoctl node status" when: '{{ kube_network_plugin == "calico" }}' @@ -97,8 +97,11 @@ - /var/log/dmesg environment: - ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem" - ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem" + ETCDCTL_API: 3 + ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem" + ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem" + ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem" + ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}" tasks: - name: set etcd_access_addresses