diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 41c87a57d87851a9b18a36b08e0e786b9bf01e74..0fe0dd5e5a9b6f92b4cd7a006301083f6e8f4c1f 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -68,7 +68,7 @@ dns_setup: true dns_domain: "{{ cluster_name }}" # # # Ip address of the kubernetes dns service -dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}" +dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" # For multi masters architecture: # kube-proxy doesn't support multiple apiservers for the time being so you'll need to configure your own loadbalancer diff --git a/roles/apps/k8s-kube-logstash b/roles/apps/k8s-kube-logstash index 256fa156e46d623ab0a7a60efdc7bac535cea8d7..340d1a5ec75e7b7c43783dc7a1c02aa7d5991dbe 160000 --- a/roles/apps/k8s-kube-logstash +++ b/roles/apps/k8s-kube-logstash @@ -1 +1 @@ -Subproject commit 256fa156e46d623ab0a7a60efdc7bac535cea8d7 +Subproject commit 340d1a5ec75e7b7c43783dc7a1c02aa7d5991dbe diff --git a/roles/etcd/tasks/install.yml b/roles/etcd/tasks/install.yml index ac3522f4d7bbd12ab1d12728d56bd90722613fc2..8d442e6b3af81784c8d62860499ff750d983c329 100644 --- a/roles/etcd/tasks/install.yml +++ b/roles/etcd/tasks/install.yml @@ -15,3 +15,9 @@ - name: Create etcd2 binary symlink file: src=/usr/local/bin/etcd dest=/usr/local/bin/etcd2 state=link + +- name: install required python module 'httplib2' + apt: + name: "python-httplib2" + state: present + when: inventory_hostname == groups['kube-master'][0] or inventory_hostname == groups['etcd'][0] diff --git a/roles/etcd/templates/etcd2-environment.j2 b/roles/etcd/templates/etcd2-environment.j2 index e9546be5ea3b8094738deaf0bc1afc0532b92b3a..2c0760388dbe7d90071a0fe22c0e8f36d9e2a32c 100644 --- a/roles/etcd/templates/etcd2-environment.j2 +++ b/roles/etcd/templates/etcd2-environment.j2 @@ -15,6 +15,6 @@ ETCD_LISTEN_CLIENT_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ETCD_LISTEN_PEER_URLS="http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2380" ETCD_NAME="{{ etcd.name }}" {% else %} -ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ host }}:2380{% if not loop.last %},{% endif %}{% endfor %}" +ETCD_INITIAL_CLUSTER="{% for host in groups['etcd'] %}master{{ loop.index|string }}=http://{{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}:2380{% if not loop.last %},{% endif %}{% endfor %}" ETCD_LISTEN_CLIENT_URLS="http://127.0.0.1:23799" {% endif %} diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index 987c41c6e599fe14f65fa6945692b9e6f44e056f..ad2739d00408907a86b0fe8e1d84401ac5513aa8 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -32,6 +32,7 @@ - "{{ kube_cert_dir }}" - "{{ kube_users_dir }}" delegate_to: "{{ groups['kube-master'][0] }}" + when: inventory_hostname != "{{ groups['kube-master'][0] }}" # Write manifests - name: Write kube-apiserver manifest @@ -48,12 +49,6 @@ port: "{{kube_apiserver_insecure_port}}" delay: 10 -- name: install required python module 'httplib2' - apt: - name: "python-httplib2" - state: present - when: inventory_hostname == groups['kube-master'][0] - - name: Create 'kube-system' namespace uri: url: http://127.0.0.1:{{ kube_apiserver_insecure_port }}/api/v1/namespaces diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index 940ec1ace518bf6fa44ff0c6efaef4303660598e..0d8cfb026ab2de716f506cd3cdef9bd3cbf051b4 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -10,7 +10,6 @@ spec: command: - /hyperkube - apiserver - - --insecure-bind-address=0.0.0.0 - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %} - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota diff --git a/roles/kubernetes/node/files/kube-gen-token.sh b/roles/kubernetes/node/files/kube-gen-token.sh index fa6a5ddc7523de59f4f0c451340c6c9ed017982c..121b52263b9773cbbb0858d1ed6cdb7b287318ca 100644 --- a/roles/kubernetes/node/files/kube-gen-token.sh +++ b/roles/kubernetes/node/files/kube-gen-token.sh @@ -19,7 +19,10 @@ token_file="${token_dir}/known_tokens.csv" create_accounts=($@) -touch "${token_file}" +if [ ! -e "${token_file}" ]; then + touch "${token_file}" +fi + for account in "${create_accounts[@]}"; do if grep ",${account}," "${token_file}" ; then continue diff --git a/roles/kubernetes/node/tasks/gen_tokens.yml b/roles/kubernetes/node/tasks/gen_tokens.yml index f2e5625f95b4fcae9bc10691ae95ac70e0054be8..7d1ce0156d525493a6109f4414ec3284589b003f 100644 --- a/roles/kubernetes/node/tasks/gen_tokens.yml +++ b/roles/kubernetes/node/tasks/gen_tokens.yml @@ -4,6 +4,7 @@ src=kube-gen-token.sh dest={{ kube_script_dir }} mode=u+x + when: inventory_hostname == groups['kube-master'][0] - name: tokens | generate tokens for master components command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" @@ -14,6 +15,7 @@ - "{{ groups['kube-master'] }}" register: gentoken changed_when: "'Added' in gentoken.stdout" + when: inventory_hostname == groups['kube-master'][0] - name: tokens | generate tokens for node components command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" @@ -24,3 +26,30 @@ - "{{ groups['kube-node'] }}" register: gentoken changed_when: "'Added' in gentoken.stdout" + when: inventory_hostname == groups['kube-master'][0] + +- name: tokens | generate tokens for calico + command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}" + environment: + TOKEN_DIR: "{{ kube_token_dir }}" + with_nested: + - [ "system:calico" ] + - "{{ groups['k8s-cluster'] }}" + register: gentoken + changed_when: "'Added' in gentoken.stdout" + when: kube_network_plugin == "calico" + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: tokens | get the calico token values + slurp: + src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token" + register: calico_token + when: kube_network_plugin == "calico" + delegate_to: "{{ groups['kube-master'][0] }}" + +- name: tokens | Add KUBE_AUTH_TOKEN for calico + lineinfile: + regexp: "^KUBE_AUTH_TOKEN=.*$" + line: "KUBE_AUTH_TOKEN={{ calico_token.content|b64decode }}" + dest: "/etc/network-environment" + when: kube_network_plugin == "calico" diff --git a/roles/kubernetes/node/tasks/secrets.yml b/roles/kubernetes/node/tasks/secrets.yml index 3d0c76734d5f23dbbff89047876da67ddb340927..4b2c208029601a96e6bd60e19d374a85671844a5 100644 --- a/roles/kubernetes/node/tasks/secrets.yml +++ b/roles/kubernetes/node/tasks/secrets.yml @@ -18,8 +18,6 @@ when: inventory_hostname == groups['kube-master'][0] - include: gen_tokens.yml - run_once: true - when: inventory_hostname == groups['kube-master'][0] # Sync certs between nodes - user: @@ -50,3 +48,4 @@ - "{{ kube_cert_dir}}/node.pem" - "{{ kube_cert_dir}}/node-key.pem" delegate_to: "{{ groups['kube-master'][0] }}" + when: inventory_hostname not in "{{ groups['kube-master'] }}" diff --git a/roles/network_plugin/tasks/calico.yml b/roles/network_plugin/tasks/calico.yml index 1ba00f6fef230b3a8cc93fef7af547e1bbbc8468..2cf3e2b2c1c6820be0ac4da1365c5857ec8c41c0 100644 --- a/roles/network_plugin/tasks/calico.yml +++ b/roles/network_plugin/tasks/calico.yml @@ -1,18 +1,46 @@ --- - name: Calico | Install calicoctl bin copy: - src={{ local_release_dir }}/calico/bin/calicoctl - dest={{ bin_dir }} - mode=0755 + src: "{{ local_release_dir }}/calico/bin/calicoctl" + dest: "{{ bin_dir }}" + mode: 0755 notify: restart calico-node - name: Calico | Create calicoctl symlink (needed by kubelet) - file: src=/usr/local/bin/calicoctl dest=/usr/bin/calicoctl state=link + file: + src: /usr/local/bin/calicoctl + dest: /usr/bin/calicoctl + state: link -- name: Calico | Configure calico-node desired pool +- name: Calico | Check if calico network pool has already been configured + uri: + url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool" + return_content: yes + status_code: 200,404 + register: calico_conf + run_once: true + delegate_to: "{{ groups['etcd'][0] }}" + +- name: Calico | Configure calico network pool shell: calicoctl pool add {{ kube_pods_subnet }} - environment: - ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" + run_once: true + when: calico_conf.status == 404 + delegate_to: "{{ groups['etcd'][0] }}" + +- name: Calico | Get calico configuration from etcd + uri: + url: "http://127.0.0.1:2379/v2/keys/calico/v1/ipam/v4/pool" + return_content: yes + register: calico_pools + run_once: true + delegate_to: "{{ groups['etcd'][0] }}" + +- name: Calico | Check if calico pool is properly configured + fail: + msg: 'Only one network pool must be configured and it must be the subnet {{ kube_pods_subnet }}. + Please erase calico configuration and run the playbook again ("etcdctl rm --recursive /calico/v1/ipam/v4/pool")' + when: ( calico_pools.json['node']['nodes'] | length > 1 ) or + ( not calico_pools.json['node']['nodes'][0]['key'] | search(".*{{ kube_pods_subnet | ipaddr('network') }}.*") ) run_once: true delegate_to: "{{ groups['etcd'][0] }}" @@ -33,9 +61,13 @@ - name: Calico | Disable node mesh shell: calicoctl bgp node-mesh off + environment: + ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" when: peer_with_router|default(false) and inventory_hostname in groups['kube-node'] - name: Calico | Configure peering with router(s) shell: calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }} + environment: + ETCD_AUTHORITY: "{{ groups['etcd'][0] }}:2379" with_items: peers when: peer_with_router|default(false) and inventory_hostname in groups['kube-node'] diff --git a/roles/network_plugin/tasks/main.yml b/roles/network_plugin/tasks/main.yml index e3ebf305ff372ad083c620ad8362504138ee785d..68363b53cee841a2c25ad303eca400ca10bbd63d 100644 --- a/roles/network_plugin/tasks/main.yml +++ b/roles/network_plugin/tasks/main.yml @@ -5,7 +5,7 @@ kube_network_plugin is not defined - name: Write network-environment - template: src=network-environment.j2 dest=/etc/network-environment mode=u+x + template: src=network-environment.j2 dest=/etc/network-environment mode=640 - include: flannel.yml when: kube_network_plugin == "flannel" diff --git a/roles/network_plugin/templates/network-environment.j2 b/roles/network_plugin/templates/network-environment.j2 index b926c8cf295e9baee863a0d37a284b17b8f497e1..6173a7a97d947347aac183cc7fd1e6f2cda1036e 100755 --- a/roles/network_plugin/templates/network-environment.j2 +++ b/roles/network_plugin/templates/network-environment.j2 @@ -5,7 +5,11 @@ CALICO_IPAM=true DEFAULT_IPV4={{ip | default(ansible_default_ipv4.address) }} # The kubernetes master IP +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +KUBERNETES_MASTER=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }} +{% else %} KUBERNETES_MASTER={{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }} +{% endif %} # Location of etcd cluster used by Calico. By default, this uses the etcd # instance running on the Kubernetes Master @@ -16,7 +20,11 @@ ETCD_AUTHORITY="127.0.0.1:23799" {% endif %} # The kubernetes-apiserver location - used by the calico plugin -KUBE_API_ROOT=http://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_insecure_port}}/api/v1/ +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +KUBE_API_ROOT=https://{{ apiserver_loadbalancer_domain_name }}:{{ loadbalancer_apiserver.port }}/api/v1/ +{% else %} +KUBE_API_ROOT=https://{{ hostvars[groups['kube-master'][0]]['ip'] | default(hostvars[groups['kube-master'][0]]['ansible_default_ipv4']['address']) }}:{{kube_apiserver_port}}/api/v1/ +{% endif %} {% else %} FLANNEL_ETCD_PREFIX="--etcd-prefix=/{{ cluster_name }}/network" {% endif %}