diff --git a/roles/etcd/tasks/install_docker.yml b/roles/etcd/tasks/install_docker.yml index f0b2779815ce12bd230896ccc4104edbca4664ee..43f4e44d886343dbedf0ffb4b39ecd1616a414f9 100644 --- a/roles/etcd/tasks/install_docker.yml +++ b/roles/etcd/tasks/install_docker.yml @@ -2,7 +2,7 @@ - name: Install | Copy etcdctl binary from docker container command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy; {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && - {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:{{ etcd_container_bin_dir }}etcdctl {{ bin_dir }}/etcdctl && + {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl && {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy" when: etcd_deployment_type == "docker" register: etcd_task_result diff --git a/roles/etcd/tasks/install_rkt.yml b/roles/etcd/tasks/install_rkt.yml index 85f8753838cd8518b9b1af773db36829b549ea43..5f700422989a491152f71a401e667641770a1a0b 100644 --- a/roles/etcd/tasks/install_rkt.yml +++ b/roles/etcd/tasks/install_rkt.yml @@ -18,7 +18,7 @@ --mount=volume=bin-dir,target=/host/bin {{ etcd_image_repo }}:{{ etcd_image_tag }} --name=etcdctl-binarycopy - --exec=/bin/cp -- {{ etcd_container_bin_dir }}/etcdctl /host/bin/etcdctl + --exec=/bin/cp -- /usr/local/bin/etcdctl /host/bin/etcdctl register: etcd_task_result until: etcd_task_result.rc == 0 retries: 4 diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2 index 11f8f74e304747db2433cb8ad6e2192f055b86e7..9ac08e0738a1cded413ae21d282357f66db9e2ba 100644 --- a/roles/etcd/templates/etcd.j2 +++ b/roles/etcd/templates/etcd.j2 @@ -17,7 +17,5 @@ {% endif %} --name={{ etcd_member_name | default("etcd") }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ - {% if etcd_after_v3 %} - {{ etcd_container_bin_dir }}etcd \ - {% endif %} + /usr/local/bin/etcd \ "$@" diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 index ca17114630ea21bdb5bfccfcc2ffb6afc3a6b9ba..d715358c8c13eb5ef0b503c9164bbf8668858edf 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-policy-controller.yml.j2 @@ -40,7 +40,7 @@ spec: memory: {{ calico_policy_controller_memory_requests }} env: - name: ETCD_ENDPOINTS - value: "{{ etcd_access_endpoint }}" + value: "{{ etcd_access_addresses }}" - name: ETCD_CA_CERT_FILE value: "{{ calico_cert_dir }}/ca_cert.crt" - name: ETCD_CERT_FILE diff --git a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 index c8dfd95249f3a18280d00cdeda105918e64e551e..78d94d31e136b148bb8a21bb2c8d17b68279bc93 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.yaml.j2 @@ -5,7 +5,7 @@ api: bindPort: {{ kube_apiserver_port }} etcd: endpoints: -{% for endpoint in etcd_access_endpoint.split(',') %} +{% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 58c7629610fef50c9fac47db38ee01d3f0fbdc2e..cad57b5f28f95f7a79356e46950e6ffd707f784c 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -29,7 +29,7 @@ spec: - /hyperkube - apiserver - --advertise-address={{ ip | default(ansible_default_ipv4.address) }} - - --etcd-servers={{ etcd_access_endpoint }} + - --etcd-servers={{ etcd_access_addresses }} - --etcd-quorum-read=true - --etcd-cafile={{ etcd_cert_dir }}/ca.pem - --etcd-certfile={{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index 96ec25499ca441f4cc7a16e6647714e981126ff6..073033315d36f8f580e22e2d30195928b35ae437 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -1,92 +1,4 @@ --- -- set_fact: - kube_apiserver_count: "{{ groups['kube-master'] | length }}" - -- set_fact: - kube_apiserver_address: "{{ ip | default(ansible_default_ipv4['address']) }}" - -- set_fact: - kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" - -- set_fact: - is_kube_master: "{{ inventory_hostname in groups['kube-master'] }}" - -- set_fact: - first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}" - -- set_fact: - loadbalancer_apiserver_localhost: false - when: loadbalancer_apiserver is defined - -- set_fact: - kube_apiserver_endpoint: |- - {% if not is_kube_master and loadbalancer_apiserver_localhost|default(true) -%} - https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }} - {%- elif is_kube_master -%} - https://127.0.0.1:{{ kube_apiserver_port }} - {%- else -%} - {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%} - https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} - {%- else -%} - https://{{ first_kube_master }}:{{ kube_apiserver_port }} - {%- endif -%} - {%- endif %} - -- set_fact: - kube_apiserver_insecure_endpoint: >- - http://{{ kube_apiserver_insecure_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_insecure_port }} - -- set_fact: - etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}" - -- set_fact: - etcd_access_address: "{{ access_ip | default(etcd_address) }}" - -- set_fact: - etcd_peer_url: "https://{{ etcd_access_address }}:2380" - -- set_fact: - etcd_client_url: "https://{{ etcd_access_address }}:2379" - -- set_fact: - etcd_authority: "127.0.0.1:2379" - -- set_fact: - etcd_endpoint: "https://{{ etcd_authority }}" - -- set_fact: - etcd_access_addresses: |- - {% for item in groups['etcd'] -%} - https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %} - {%- endfor %} - -- set_fact: - etcd_access_endpoint: "{% if etcd_multiaccess|default(true) %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}" - -- set_fact: - etcd_member_name: |- - {% for host in groups['etcd'] %} - {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} - {% endfor %} - -- set_fact: - etcd_peer_addresses: |- - {% for item in groups['etcd'] -%} - {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} - {%- endfor %} - -- set_fact: - is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" - -- set_fact: - etcd_after_v3: etcd_version | version_compare("v3.0.0", ">=") - -- set_fact: - etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" - -- set_fact: - peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" - - name: check if atomic host stat: path: /run/ostree-booted diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 6e84a0311e91bb4622b4caad1665b69ac1ff20c3..61f820c627e1f6b5bd92f9d439fc8275929de86f 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -77,6 +77,9 @@ kube_users: # Can also be set to 'cloud', which lets the cloud provider setup appropriate routing kube_network_plugin: calico +# Determines if calico-rr group exists +peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" + # Kubernetes internal network for services, unused block of space. kube_service_addresses: 10.233.0.0/18 @@ -158,3 +161,44 @@ vault_cert_dir: "{{ vault_base_dir }}/ssl" vault_config_dir: "{{ vault_base_dir }}/config" vault_roles_dir: "{{ vault_base_dir }}/roles" vault_secrets_dir: "{{ vault_base_dir }}/secrets" + +# Vars for pointing to kubernetes api endpoints +is_kube_master: "{{ inventory_hostname in groups['kube-master'] }}" +kube_apiserver_count: "{{ groups['kube-master'] | length }}" +kube_apiserver_address: "{{ ip | default(ansible_default_ipv4['address']) }}" +kube_apiserver_access_address: "{{ access_ip | default(kube_apiserver_address) }}" +first_kube_master: "{{ hostvars[groups['kube-master'][0]]['access_ip'] | default(hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address'])) }}" +loadbalancer_apiserver_localhost: "{{ loadbalancer_apiserver is not defined }}" +kube_apiserver_endpoint: |- + {% if not is_kube_master and loadbalancer_apiserver_localhost|default(true) -%} + https://localhost:{{ nginx_kube_apiserver_port|default(kube_apiserver_port) }} + {%- elif is_kube_master -%} + https://127.0.0.1:{{ kube_apiserver_port }} + {%- else -%} + {%- if loadbalancer_apiserver is defined and loadbalancer_apiserver.port is defined -%} + https://{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}:{{ loadbalancer_apiserver.port|default(kube_apiserver_port) }} + {%- else -%} + https://{{ first_kube_master }}:{{ kube_apiserver_port }} + {%- endif -%} + {%- endif %} +kube_apiserver_insecure_endpoint: >- + http://{{ kube_apiserver_insecure_bind_address | regex_replace('0\.0\.0\.0','127.0.0.1') }}:{{ kube_apiserver_insecure_port }} + +# Vars for pointing to etcd endpoints +is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}" +etcd_address: "{{ ip | default(ansible_default_ipv4['address']) }}" +etcd_access_address: "{{ access_ip | default(etcd_address) }}" +etcd_peer_url: "https://{{ etcd_access_address }}:2380" +etcd_client_url: "https://{{ etcd_access_address }}:2379" +etcd_access_addresses: |- + {% for item in groups['etcd'] -%} + https://{{ hostvars[item]['access_ip'] | default(hostvars[item]['ip'] | default(hostvars[item]['ansible_default_ipv4']['address'])) }}:2379{% if not loop.last %},{% endif %} + {%- endfor %} +etcd_member_name: |- + {% for host in groups['etcd'] %} + {% if inventory_hostname == host %}{{"etcd"+loop.index|string }}{% endif %} + {% endfor %} +etcd_peer_addresses: |- + {% for item in groups['etcd'] -%} + {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %} + {%- endfor %} diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 index 201caecfe897ff7bfbc22c0a7745358b449a3401..1cdb2659c4b39b1e2ad177282055589d9f57bb6a 100644 --- a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 +++ b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 @@ -1,4 +1,4 @@ -ETCD_ENDPOINTS="{{ etcd_access_endpoint }}" +ETCD_ENDPOINTS="{{ etcd_access_addresses }}" ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt" ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt" ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem" diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2 index a4207f1dc712096132a9408bbae20f1f572a8b0d..fbae4eda1de5b22cb68862e0fe4ea71d152d4778 100644 --- a/roles/network_plugin/calico/templates/calico-config.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-config.yml.j2 @@ -4,7 +4,7 @@ metadata: name: calico-config namespace: {{ system_namespace }} data: - etcd_endpoints: "{{ etcd_access_endpoint }}" + etcd_endpoints: "{{ etcd_access_addresses }}" etcd_ca: "/calico-secrets/ca_cert.crt" etcd_cert: "/calico-secrets/cert.crt" etcd_key: "/calico-secrets/key.pem" diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2 index b752fe340537c80c4509fbf11aa959ac28839a3d..c9a1b5d4072a97247ed074358587bb86e96d32dd 100644 --- a/roles/network_plugin/calico/templates/calicoctl-container.j2 +++ b/roles/network_plugin/calico/templates/calicoctl-container.j2 @@ -1,7 +1,7 @@ #!/bin/bash {{ docker_bin_dir }}/docker run -i --privileged --rm \ --net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ +-e ETCD_ENDPOINTS={{ etcd_access_addresses }} \ -e ETCD_CA_CERT_FILE={{ calico_cert_dir }}/ca_cert.crt \ -e ETCD_CERT_FILE={{ calico_cert_dir }}/cert.crt \ -e ETCD_KEY_FILE={{ calico_cert_dir }}/key.pem \ diff --git a/roles/network_plugin/calico/templates/cni-calico.conf.j2 b/roles/network_plugin/calico/templates/cni-calico.conf.j2 index 49be7e2ac171341f8148213cbcad8283842ebe6f..892391d11c67a9ad71ba99f1b806cd533a139d8b 100644 --- a/roles/network_plugin/calico/templates/cni-calico.conf.j2 +++ b/roles/network_plugin/calico/templates/cni-calico.conf.j2 @@ -6,7 +6,7 @@ "nodename": "{{ ansible_hostname }}", {% endif %} "type": "calico", - "etcd_endpoints": "{{ etcd_access_endpoint }}", + "etcd_endpoints": "{{ etcd_access_addresses }}", "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem", "etcd_key_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem", "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem", diff --git a/roles/network_plugin/canal/templates/calicoctl-container.j2 b/roles/network_plugin/canal/templates/calicoctl-container.j2 index 28f09b4c56f926d10cfd0bcc9c5b961b4a9a9273..cc0a38bfc04d37944ac022979411ca755dc0e1c2 100644 --- a/roles/network_plugin/canal/templates/calicoctl-container.j2 +++ b/roles/network_plugin/canal/templates/calicoctl-container.j2 @@ -1,7 +1,7 @@ #!/bin/bash {{ docker_bin_dir }}/docker run -i --privileged --rm \ --net=host --pid=host \ --e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \ +-e ETCD_ENDPOINTS={{ etcd_access_addresses }} \ -e ETCD_CA_CERT_FILE={{ canal_cert_dir }}/ca_cert.crt \ -e ETCD_CERT_FILE={{ canal_cert_dir }}/cert.crt \ -e ETCD_KEY_FILE={{ canal_cert_dir }}/key.pem \ diff --git a/roles/network_plugin/canal/templates/canal-config.yaml.j2 b/roles/network_plugin/canal/templates/canal-config.yaml.j2 index 1d0d3002a33cdfc2a72bf3c0af9e3a05219dcaad..ab6c276bd64162903a1fba165ca01c6dde2e8950 100644 --- a/roles/network_plugin/canal/templates/canal-config.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-config.yaml.j2 @@ -7,7 +7,7 @@ metadata: name: canal-config data: # Configure this with the location of your etcd cluster. - etcd_endpoints: "{{ etcd_access_endpoint }}" + etcd_endpoints: "{{ etcd_access_addresses }}" # The interface used by canal for host <-> host communication. # If left blank, then the interface is chosing using the node's diff --git a/roles/network_plugin/canal/templates/cni-canal.conf.j2 b/roles/network_plugin/canal/templates/cni-canal.conf.j2 index b835443c79869099d70d231bdd71f9d349e7844a..b47d7f9dd1ce025afd3981716ac13d500b3458dc 100644 --- a/roles/network_plugin/canal/templates/cni-canal.conf.j2 +++ b/roles/network_plugin/canal/templates/cni-canal.conf.j2 @@ -3,7 +3,7 @@ "type": "flannel", "delegate": { "type": "calico", - "etcd_endpoints": "{{ etcd_access_endpoint }}", + "etcd_endpoints": "{{ etcd_access_addresses }}", "log_level": "info", "policy": { "type": "k8s"