From a32cd85eb7f0fa4c70ead3670fbc73d9e541e26a Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Wed, 9 Nov 2016 13:44:41 +0300
Subject: [PATCH] Add etcd TLS support
---
roles/download/defaults/main.yml | 5 +-
roles/etcd/defaults/main.yml | 6 +
roles/etcd/files/make-ssl-etcd.sh | 80 +++++++++++++
roles/etcd/handlers/main.yml | 7 +-
roles/etcd/tasks/check_certs.yml | 36 ++++++
roles/etcd/tasks/gen_certs.yml | 111 ++++++++++++++++++
roles/etcd/tasks/main.yml | 2 +
roles/etcd/tasks/pre_upgrade.yml | 34 ++++++
roles/etcd/templates/deb-etcd-docker.initd.j2 | 3 +-
roles/etcd/templates/etcd-docker.service.j2 | 3 +-
roles/etcd/templates/etcd.j2 | 13 +-
roles/etcd/templates/openssl.conf.j2 | 39 ++++++
.../templates/calico-policy-controller.yml.j2 | 6 +
roles/kubernetes/master/defaults/main.yml | 6 +
.../manifests/kube-apiserver.manifest.j2 | 10 +-
.../node/templates/cni-calico.conf.j2 | 3 +
.../kubernetes/preinstall/tasks/set_facts.yml | 10 +-
roles/network_plugin/calico/defaults/main.yml | 3 +
roles/network_plugin/calico/tasks/main.yml | 26 +++-
.../calico/templates/calicoctl-container.j2 | 4 +
.../calico/templates/network-environment.j2 | 3 +
roles/network_plugin/flannel/tasks/main.yml | 12 +-
.../flannel/templates/flannel-pod.yml | 18 +--
.../flannel/templates/network.json | 1 -
roles/uploads/defaults/main.yml | 2 +-
25 files changed, 408 insertions(+), 35 deletions(-)
create mode 100755 roles/etcd/files/make-ssl-etcd.sh
create mode 100644 roles/etcd/tasks/check_certs.yml
create mode 100644 roles/etcd/tasks/gen_certs.yml
create mode 100644 roles/etcd/tasks/pre_upgrade.yml
create mode 100644 roles/etcd/templates/openssl.conf.j2
delete mode 100644 roles/network_plugin/flannel/templates/network.json
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index cbe053fa0..8f33ebd15 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -10,7 +10,7 @@ kube_version: v1.4.3
etcd_version: v3.0.6
#TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
# after migration to container download
-calico_version: v0.22.0
+calico_version: v1.0.0-beta
calico_cni_version: v1.4.2
weave_version: v1.6.1
flannel_version: v0.6.2
@@ -39,7 +39,8 @@ flannel_server_helper_image_tag: "{{ flannel_server_helper_version }}"
flannel_image_repo: "quay.io/coreos/flannel"
flannel_image_tag: "{{ flannel_version }}"
calicoctl_image_repo: "calico/ctl"
-calicoctl_image_tag: "{{ calico_version }}"
+# TODO(mattymo): v1.0.0-beta has different syntax. Needs work to upgrade
+calicoctl_image_tag: "v0.22.0"
calico_node_image_repo: "calico/node"
calico_node_image_tag: "{{ calico_version }}"
hyperkube_image_repo: "quay.io/coreos/hyperkube"
diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml
index 02234a2fe..2df4ba165 100644
--- a/roles/etcd/defaults/main.yml
+++ b/roles/etcd/defaults/main.yml
@@ -1,2 +1,8 @@
---
etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/"
+
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+etcd_cert_group: root
+
+etcd_script_dir: "{{ bin_dir }}/etcd-scripts"
diff --git a/roles/etcd/files/make-ssl-etcd.sh b/roles/etcd/files/make-ssl-etcd.sh
new file mode 100755
index 000000000..4c7db9430
--- /dev/null
+++ b/roles/etcd/files/make-ssl-etcd.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Author: Smana smainklh@gmail.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+
+usage()
+{
+ cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+ -h | --help : Show this message
+ -f | --config : Openssl configuration file
+ -d | --ssldir : Directory where the certificates will be installed
+
+ ex :
+ $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+ case "$1" in
+ -h | --help) usage; exit 0;;
+ -f | --config) CONFIG=${2}; shift 2;;
+ -d | --ssldir) SSLDIR="${2}"; shift 2;;
+ *)
+ usage
+ echo "ERROR : Unknown option"
+ exit 3
+ ;;
+ esac
+done
+
+if [ -z ${CONFIG} ]; then
+ echo "ERROR: the openssl configuration file is missing. option -f"
+ exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+ SSLDIR="/etc/ssl/etcd"
+fi
+
+tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
+
+# ETCD member
+openssl genrsa -out member-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key member-key.pem -out member.csr -subj "/CN=etcd-member" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in member.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+# Nodes and Admin
+for i in node admin; do
+ openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+ openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+ openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+done
+
+# Install certs
+mv *.pem ${SSLDIR}/
diff --git a/roles/etcd/handlers/main.yml b/roles/etcd/handlers/main.yml
index 8104ff1a8..badf0bd79 100644
--- a/roles/etcd/handlers/main.yml
+++ b/roles/etcd/handlers/main.yml
@@ -11,7 +11,7 @@
when: ansible_service_mgr == "systemd"
- name: wait for etcd up
- uri: url="http://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health"
+ uri: url="https://{% if is_etcd_master %}{{ etcd_address }}{% else %}127.0.0.1{% endif %}:2379/health" validate_certs=no
register: result
until: result.status is defined and result.status == 200
retries: 10
@@ -22,3 +22,8 @@
name: etcd
state: restarted
when: is_etcd_master
+
+- name: set etcd_secret_changed
+ set_fact:
+ etcd_secret_changed: true
+
diff --git a/roles/etcd/tasks/check_certs.yml b/roles/etcd/tasks/check_certs.yml
new file mode 100644
index 000000000..03a875517
--- /dev/null
+++ b/roles/etcd/tasks/check_certs.yml
@@ -0,0 +1,36 @@
+---
+- name: "Check_certs | check if the certs have already been generated on first master"
+ stat:
+ path: "{{ etcd_cert_dir }}/ca.pem"
+ delegate_to: "{{groups['etcd'][0]}}"
+ register: etcdcert_master
+ run_once: true
+
+- name: "Check_certs | Set default value for 'sync_certs' and 'gen_certs' to false"
+ set_fact:
+ sync_certs: false
+ gen_certs: false
+
+- name: "Check_certs | Set 'sync_certs' and 'gen_certs' to true"
+ set_fact:
+ gen_certs: true
+ when: not etcdcert_master.stat.exists
+ run_once: true
+
+- name: "Check certs | check if a cert already exists"
+ stat:
+ path: "{{ etcd_cert_dir }}/ca.pem"
+ register: etcdcert
+
+- name: "Check_certs | Set 'sync_certs' to true"
+ set_fact:
+ sync_certs: true
+ when: >-
+ {%- set certs = {'sync': False} -%}
+ {%- for server in play_hosts
+ if (not hostvars[server].etcdcert.stat.exists|default(False)) or
+ (hostvars[server].etcdcert.stat.checksum|default('') != etcdcert_master.stat.checksum|default('')) -%}
+ {%- set _ = certs.update({'sync': True}) -%}
+ {%- endfor -%}
+ {{ certs.sync }}
+ run_once: true
diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs.yml
new file mode 100644
index 000000000..e0aad58a2
--- /dev/null
+++ b/roles/etcd/tasks/gen_certs.yml
@@ -0,0 +1,111 @@
+---
+
+- name: Gen_certs | create etcd script dir
+ file:
+ path: "{{ etcd_script_dir }}"
+ state: directory
+ owner: root
+ when: inventory_hostname == groups['etcd'][0]
+
+- name: Gen_certs | create etcd cert dir
+ file:
+ path={{ etcd_cert_dir }}
+ group={{ etcd_cert_group }}
+ state=directory
+ owner=root
+ recurse=yes
+
+- name: Gen_certs | write openssl config
+ template:
+ src: "openssl.conf.j2"
+ dest: "{{ etcd_config_dir }}/openssl.conf"
+ run_once: yes
+ delegate_to: "{{groups['etcd'][0]}}"
+ when: gen_certs|default(false)
+
+- name: Gen_certs | copy certs generation script
+ copy:
+ src: "make-ssl-etcd.sh"
+ dest: "{{ etcd_script_dir }}/make-ssl-etcd.sh"
+ mode: 0700
+ run_once: yes
+ delegate_to: "{{groups['etcd'][0]}}"
+ when: gen_certs|default(false)
+
+- name: Gen_certs | run cert generation script
+ command: "{{ etcd_script_dir }}/make-ssl-etcd.sh -f {{ etcd_config_dir }}/openssl.conf -d {{ etcd_cert_dir }}"
+ run_once: yes
+ delegate_to: "{{groups['etcd'][0]}}"
+ when: gen_certs|default(false)
+ notify: set etcd_secret_changed
+
+- set_fact:
+ master_certs: ['ca-key.pem', 'admin.pem', 'admin-key.pem', 'member.pem', 'member-key.pem']
+ node_certs: ['ca.pem', 'node.pem', 'node-key.pem']
+
+- name: Gen_certs | Gather etcd master certs
+ shell: "tar cfz - -C {{ etcd_cert_dir }} {{ master_certs|join(' ') }} {{ node_certs|join(' ') }}| base64 --wrap=0"
+ register: etcd_master_cert_data
+ delegate_to: "{{groups['etcd'][0]}}"
+ run_once: true
+ when: sync_certs|default(false)
+ notify: set etcd_secret_changed
+
+- name: Gen_certs | Gather etcd node certs
+ shell: "tar cfz - -C {{ etcd_cert_dir }} {{ node_certs|join(' ') }} | base64 --wrap=0"
+ register: etcd_node_cert_data
+ delegate_to: "{{groups['etcd'][0]}}"
+ run_once: true
+ when: sync_certs|default(false)
+ notify: set etcd_secret_changed
+
+- name: Gen_certs | Copy certs on masters
+ shell: "echo '{{etcd_master_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
+ changed_when: false
+ when: inventory_hostname in groups['etcd'] and sync_certs|default(false) and
+ inventory_hostname != groups['etcd'][0]
+
+- name: Gen_certs | Copy certs on nodes
+ shell: "echo '{{etcd_node_cert_data.stdout|quote}}' | base64 -d | tar xz -C {{ etcd_cert_dir }}"
+ changed_when: false
+ when: inventory_hostname in groups['k8s-cluster'] and sync_certs|default(false) and
+ inventory_hostname not in groups['etcd']
+
+- name: Gen_certs | check certificate permissions
+ file:
+ path={{ etcd_cert_dir }}
+ group={{ etcd_cert_group }}
+ state=directory
+ owner=kube
+ recurse=yes
+
+- name: Gen_certs | set permissions on keys
+ shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem
+ when: inventory_hostname in groups['etcd']
+ changed_when: false
+
+- name: Gen_certs | target ca-certificates directory
+ set_fact:
+ ca_cert_dir: |-
+ {% if ansible_os_family == "Debian" -%}
+ /usr/local/share/ca-certificates
+ {%- elif ansible_os_family == "RedHat" -%}
+ /etc/pki/ca-trust/source/anchors
+ {%- elif ansible_os_family == "CoreOS" -%}
+ /etc/ssl/certs
+ {%- endif %}
+
+- name: Gen_certs | add CA to trusted CA dir
+ copy:
+ src: "{{ etcd_cert_dir }}/ca.pem"
+ dest: "{{ ca_cert_dir }}/etcd-ca.crt"
+ remote_src: true
+ register: etcd_ca_cert
+
+- name: Gen_certs | update ca-certificates (Debian/Ubuntu/CoreOS)
+ command: update-ca-certificates
+ when: etcd_ca_cert.changed and ansible_os_family in ["Debian", "CoreOS"]
+
+- name: Gen_certs | update ca-certificatesa (RedHat)
+ command: update-ca-trust extract
+ when: etcd_ca_cert.changed and ansible_os_family == "RedHat"
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 88dfe59d8..15be1a769 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -1,5 +1,7 @@
---
- include: pre_upgrade.yml
+- include: check_certs.yml
+- include: gen_certs.yml
- include: install.yml
when: is_etcd_master
- include: set_cluster_health.yml
diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml
new file mode 100644
index 000000000..d1962ea92
--- /dev/null
+++ b/roles/etcd/tasks/pre_upgrade.yml
@@ -0,0 +1,34 @@
+- name: "Pre-upgrade | check for etcd-proxy unit file"
+ stat:
+ path: /etc/systemd/system/etcd-proxy.service
+ register: kube_apiserver_service_file
+
+- name: "Pre-upgrade | check for etcd-proxy init script"
+ stat:
+ path: /etc/init.d/etcd-proxy
+ register: kube_apiserver_init_script
+
+- name: "Pre-upgrade | stop etcd-proxy if service defined"
+ service:
+ name: etcd-proxy
+ state: stopped
+ when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+
+- name: "Pre-upgrade | remove etcd-proxy service definition"
+ file:
+ path: "{{ item }}"
+ state: absent
+ when: (kube_apiserver_service_file.stat.exists|default(False) or kube_apiserver_init_script.stat.exists|default(False))
+ with_items:
+ - /etc/systemd/system/etcd-proxy.service
+ - /etc/init.d/etcd-proxy
+
+- name: "Pre-upgrade | find etcd-proxy container"
+ command: docker ps -aq --filter "name=etcd-proxy*"
+ register: etcd_proxy_container
+ ignore_errors: true
+
+- name: "Pre-upgrade | remove etcd-proxy if it exists"
+ command: "docker rm -f {{item}}"
+ with_items: "{{etcd_proxy_container.stdout_lines}}"
+
diff --git a/roles/etcd/templates/deb-etcd-docker.initd.j2 b/roles/etcd/templates/deb-etcd-docker.initd.j2
index a83aae184..4457b37b9 100644
--- a/roles/etcd/templates/deb-etcd-docker.initd.j2
+++ b/roles/etcd/templates/deb-etcd-docker.initd.j2
@@ -19,8 +19,9 @@ DAEMON={{ docker_bin_dir | default("/usr/bin") }}/docker
DAEMON_EXEC=`basename $DAEMON`
DAEMON_ARGS="run --restart=always --env-file=/etc/etcd.env \
--net=host \
--v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /etc/ssl/certs:/etc/ssl/certs:ro \
-v /var/lib/etcd:/var/lib/etcd:rw \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
--name={{ etcd_member_name | default("etcd") }} \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
{% if etcd_after_v3 %}
diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2
index a37759fec..ff40b5b59 100644
--- a/roles/etcd/templates/etcd-docker.service.j2
+++ b/roles/etcd/templates/etcd-docker.service.j2
@@ -11,7 +11,8 @@ ExecStart={{ docker_bin_dir | default("/usr/bin") }}/docker run --restart=always
{# TODO(mattymo): Allow docker IP binding and disable in envfile
-p 2380:2380 -p 2379:2379 #}
--net=host \
--v /usr/share/ca-certificates/:/etc/ssl/certs:ro \
+-v /etc/ssl/certs:/etc/ssl/certs:ro \
+-v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \
-v /var/lib/etcd:/var/lib/etcd:rw \
--name={{ etcd_member_name | default("etcd") }} \
{{ etcd_image_repo }}:{{ etcd_image_tag }} \
diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2
index c6dc4c28b..0b7e1eb9f 100644
--- a/roles/etcd/templates/etcd.j2
+++ b/roles/etcd/templates/etcd.j2
@@ -3,10 +3,19 @@ ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
-ETCD_LISTEN_CLIENT_URLS=http://{{ etcd_address }}:2379,http://127.0.0.1:2379
+ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2379,https://127.0.0.1:2379
ETCD_ELECTION_TIMEOUT=10000
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
-ETCD_LISTEN_PEER_URLS=http://{{ etcd_address }}:2380
+ETCD_LISTEN_PEER_URLS=https://{{ etcd_address }}:2380
ETCD_NAME={{ etcd_member_name }}
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER={{ etcd_peer_addresses }}
+
+# TLS settings
+ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_CERT_FILE={{ etcd_cert_dir }}/node.pem
+ETCD_KEY_FILE={{ etcd_cert_dir }}/node-key.pem
+ETCD_PEER_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
+ETCD_PEER_CERT_FILE={{ etcd_cert_dir }}/member.pem
+ETCD_PEER_KEY_FILE={{ etcd_cert_dir }}/member-key.pem
+ETCD_PEER_CLIENT_CERT_AUTH=true
diff --git a/roles/etcd/templates/openssl.conf.j2 b/roles/etcd/templates/openssl.conf.j2
new file mode 100644
index 000000000..3ea328289
--- /dev/null
+++ b/roles/etcd/templates/openssl.conf.j2
@@ -0,0 +1,39 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+subjectAltName = @alt_names
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+authorityKeyIdentifier=keyid:always,issuer
+
+[alt_names]
+DNS.1 = localhost
+{% for host in groups['etcd'] %}
+DNS.{{ 1 + loop.index }} = {{ host }}
+{% endfor %}
+{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+{% set idx = groups['etcd'] | length | int + 1 %}
+DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
+{% endif %}
+{% for host in groups['etcd'] %}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx = groups['etcd'] | length | int * 2 + 1 %}
+IP.{{ idx }} = 127.0.0.1
diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2
index ff69b5ec6..5e0586e16 100644
--- a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2
+++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2
@@ -27,6 +27,12 @@ spec:
env:
- name: ETCD_ENDPOINTS
value: "{{ etcd_access_endpoint }}"
+ - name: ETCD_CA_CERT_FILE
+ value: "{{ etcd_cert_dir }}/ca.pem"
+ - name: ETCD_CERT_FILE
+ value: "{{ etcd_cert_dir }}/node.pem"
+ - name: ETCD_KEY_FILE
+ value: "{{ etcd_cert_dir }}/node-key.pem"
# Location of the Kubernetes API - this shouldn't need to be
# changed so long as it is used in conjunction with
# CONFIGURE_ETC_HOSTS="true".
diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml
index ee32ccf57..269ed3714 100644
--- a/roles/kubernetes/master/defaults/main.yml
+++ b/roles/kubernetes/master/defaults/main.yml
@@ -28,3 +28,9 @@ kube_apiserver_insecure_bind_address: 127.0.0.1
# Logging directory (sysvinit systems)
kube_log_dir: "/var/log/kubernetes"
+
+# ETCD cert dir for connecting apiserver to etcd
+etcd_config_dir: /etc/ssl/etcd
+etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+
+
diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index bcf9f22d4..192e6021b 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -14,6 +14,9 @@ spec:
- --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
- --etcd-servers={{ etcd_access_endpoint }}
- --etcd-quorum-read=true
+ - --etcd-cafile={{ etcd_cert_dir }}/ca.pem
+ - --etcd-certfile={{ etcd_cert_dir }}/node.pem
+ - --etcd-keyfile={{ etcd_cert_dir }}/node-key.pem
- --insecure-bind-address={{ kube_apiserver_insecure_bind_address }}
- --apiserver-count={{ kube_apiserver_count }}
- --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
@@ -50,6 +53,9 @@ spec:
- mountPath: /etc/ssl/certs
name: ssl-certs-host
readOnly: true
+ - mountPath: {{ etcd_cert_dir }}
+ name: etcd-certs
+ readOnly: true
- mountPath: /var/log/
name: logfile
volumes:
@@ -59,7 +65,9 @@ spec:
- hostPath:
path: /etc/ssl/certs/
name: ssl-certs-host
+ - hostPath:
+ path: {{ etcd_cert_dir }}
+ name: etcd-certs
- hostPath:
path: /var/log/
name: logfile
-
diff --git a/roles/kubernetes/node/templates/cni-calico.conf.j2 b/roles/kubernetes/node/templates/cni-calico.conf.j2
index a6558deaa..4e9752ef4 100644
--- a/roles/kubernetes/node/templates/cni-calico.conf.j2
+++ b/roles/kubernetes/node/templates/cni-calico.conf.j2
@@ -2,6 +2,9 @@
"name": "calico-k8s-network",
"type": "calico",
"etcd_endpoints": "{{ etcd_access_endpoint }}",
+ "etcd_cert_file": "{{ etcd_cert_dir }}/node.pem",
+ "etcd_key_file": "{{ etcd_cert_dir }}/node-key.pem",
+ "etcd_ca_cert_file": "{{ etcd_cert_dir }}/ca.pem",
"log_level": "info",
"ipam": {
"type": "calico-ipam"
diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml
index d51bcbed4..aec296c6e 100644
--- a/roles/kubernetes/preinstall/tasks/set_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_facts.yml
@@ -23,14 +23,14 @@
- set_fact: etcd_address="{{ ip | default(ansible_default_ipv4['address']) }}"
- set_fact: etcd_access_address="{{ access_ip | default(etcd_address) }}"
-- set_fact: etcd_peer_url="http://{{ etcd_access_address }}:2380"
-- set_fact: etcd_client_url="http://{{ etcd_access_address }}:2379"
+- set_fact: etcd_peer_url="https://{{ etcd_access_address }}:2380"
+- set_fact: etcd_client_url="https://{{ etcd_access_address }}:2379"
- set_fact: etcd_authority="127.0.0.1:2379"
-- set_fact: etcd_endpoint="http://{{ etcd_authority }}"
+- set_fact: etcd_endpoint="https://{{ etcd_authority }}"
- set_fact:
etcd_access_addresses: |-
{% for item in groups['etcd'] -%}
- http://{{ item }}:2379{% if not loop.last %},{% endif %}
+ https://{{ item }}:2379{% if not loop.last %},{% endif %}
{%- endfor %}
- set_fact: etcd_access_endpoint="{% if etcd_multiaccess %}{{ etcd_access_addresses }}{% else %}{{ etcd_endpoint }}{% endif %}"
- set_fact:
@@ -41,7 +41,7 @@
- set_fact:
etcd_peer_addresses: |-
{% for item in groups['etcd'] -%}
- {{ "etcd"+loop.index|string }}=http://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
+ {{ "etcd"+loop.index|string }}=https://{{ hostvars[item].access_ip | default(hostvars[item].ip | default(hostvars[item].ansible_default_ipv4['address'])) }}:2380{% if not loop.last %},{% endif %}
{%- endfor %}
- set_fact:
is_etcd_master: "{{ inventory_hostname in groups['etcd'] }}"
diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml
index aec7a5e15..7b608ab7e 100644
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@@ -8,3 +8,6 @@ ipip: false
# Set to true if you want your calico cni binaries to overwrite the
# ones from hyperkube while leaving other cni plugins intact.
overwrite_hyperkube_cni: true
+
+calico_cert_dir: /etc/calico/certs
+etcd_cert_dir: /etc/ssl/etcd/ssl
diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index ce43d5224..60a728ba0 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -12,6 +12,24 @@
- meta: flush_handlers
+- name: Calico | Create calico certs directory
+ file:
+ dest: "{{ calico_cert_dir }}"
+ state: directory
+ mode: 0750
+ owner: root
+ group: root
+
+- name: Calico | Link etcd certificates for calico-node
+ file:
+ src: "{{ etcd_cert_dir }}/{{ item.s }}"
+ dest: "{{ calico_cert_dir }}/{{ item.d }}"
+ state: hard
+ with_items:
+ - {s: "ca.pem", d: "ca_cert.crt"}
+ - {s: "node.pem", d: "cert.crt"}
+ - {s: "node-key.pem", d: "key.pem"}
+
- name: Calico | Install calicoctl container script
template:
src: calicoctl-container.j2
@@ -41,7 +59,7 @@
when: "{{ overwrite_hyperkube_cni|bool }}"
- name: Calico | wait for etcd
- uri: url=http://localhost:2379/health
+ uri: url=https://localhost:2379/health validate_certs=no
register: result
until: result.status == 200 or result.status == 401
retries: 10
@@ -87,7 +105,11 @@
- name: Calico | Get calico configuration from etcd
command: |-
- curl http://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
+ curl \
+ --cacert {{ etcd_cert_dir }}/ca.pem \
+ --cert {{ etcd_cert_dir}}/admin.pem \
+ --key {{ etcd_cert_dir }}/admin-key.pem \
+ https://localhost:2379/v2/keys/calico/v1/ipam/v4/pool
register: calico_pools_raw
delegate_to: "{{groups['etcd'][0]}}"
run_once: true
diff --git a/roles/network_plugin/calico/templates/calicoctl-container.j2 b/roles/network_plugin/calico/templates/calicoctl-container.j2
index c8ac759de..9d47c73ca 100644
--- a/roles/network_plugin/calico/templates/calicoctl-container.j2
+++ b/roles/network_plugin/calico/templates/calicoctl-container.j2
@@ -2,8 +2,12 @@
/usr/bin/docker run --privileged --rm \
--net=host --pid=host \
-e ETCD_ENDPOINTS={{ etcd_access_endpoint }} \
+-e ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt \
+-e ETCD_CERT_FILE=/etc/calico/certs/cert.crt \
+-e ETCD_KEY_FILE=/etc/calico/certs/key.pem \
-v /usr/bin/docker:/usr/bin/docker \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /var/run/calico:/var/run/calico \
+-v /etc/calico/certs:/etc/calico/certs:ro \
{{ calicoctl_image_repo }}:{{ calicoctl_image_tag}} \
$@
diff --git a/roles/network_plugin/calico/templates/network-environment.j2 b/roles/network_plugin/calico/templates/network-environment.j2
index 0da2db904..8fd13d36c 100644
--- a/roles/network_plugin/calico/templates/network-environment.j2
+++ b/roles/network_plugin/calico/templates/network-environment.j2
@@ -7,3 +7,6 @@ KUBERNETES_MASTER={{ kube_apiserver_endpoint }}
# IP and port of etcd instance used by Calico
ETCD_ENDPOINTS={{ etcd_access_endpoint }}
+ETCD_CA_CERT_FILE=/etc/calico/certs/ca_cert.crt
+ETCD_CERT_FILE=/etc/calico/certs/cert.crt
+ETCD_KEY_FILE=/etc/calico/certs/key.pem
diff --git a/roles/network_plugin/flannel/tasks/main.yml b/roles/network_plugin/flannel/tasks/main.yml
index a6fa183ef..8581d2ce7 100644
--- a/roles/network_plugin/flannel/tasks/main.yml
+++ b/roles/network_plugin/flannel/tasks/main.yml
@@ -1,9 +1,11 @@
---
-- name: Flannel | Write flannel configuration
- template:
- src: network.json
- dest: /etc/flannel-network.json
- backup: yes
+- name: Flannel | Set Flannel etcd configuration
+ command: |-
+ {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
+ set /{{ cluster_name }}/network/config \
+ '{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }'
+ delegate_to: "{{groups['etcd'][0]}}"
+ run_once: true
- name: Flannel | Create flannel pod manifest
template:
diff --git a/roles/network_plugin/flannel/templates/flannel-pod.yml b/roles/network_plugin/flannel/templates/flannel-pod.yml
index 70353f11a..02c41e18b 100644
--- a/roles/network_plugin/flannel/templates/flannel-pod.yml
+++ b/roles/network_plugin/flannel/templates/flannel-pod.yml
@@ -12,26 +12,16 @@
- name: "subnetenv"
hostPath:
path: "/run/flannel"
- - name: "networkconfig"
+ - name: "etcd-certs"
hostPath:
- path: "/etc/flannel-network.json"
+ path: "{{ etcd_cert_dir }}"
containers:
- - name: "flannel-server-helper"
- image: "{{ flannel_server_helper_image_repo }}:{{ flannel_server_helper_image_tag }}"
- args:
- - "--network-config=/etc/flannel-network.json"
- - "--etcd-prefix=/{{ cluster_name }}/network"
- - "--etcd-endpoints={{ etcd_access_endpoint }}"
- volumeMounts:
- - name: "networkconfig"
- mountPath: "/etc/flannel-network.json"
- imagePullPolicy: "Always"
- name: "flannel-container"
image: "{{ flannel_image_repo }}:{{ flannel_image_tag }}"
command:
- "/bin/sh"
- "-c"
- - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
+ - "/opt/bin/flanneld -etcd-endpoints {{ etcd_access_endpoint }} -etcd-prefix /{{ cluster_name }}/network -etcd-cafile {{ etcd_cert_dir }}/ca.pem -etcd-certfile {{ etcd_cert_dir }}/node.pem -etcd-keyfile {{ etcd_cert_dir }}/node-key.pem {% if flannel_interface is defined %}-iface {{ flannel_interface }}{% endif %} {% if flannel_public_ip is defined %}-public-ip {{ flannel_public_ip }}{% endif %}"
ports:
- hostPort: 10253
containerPort: 10253
@@ -41,6 +31,8 @@
volumeMounts:
- name: "subnetenv"
mountPath: "/run/flannel"
+ - name: "etcd-certs"
+ mountPath: "{{ etcd_cert_dir }}"
securityContext:
privileged: true
hostNetwork: true
diff --git a/roles/network_plugin/flannel/templates/network.json b/roles/network_plugin/flannel/templates/network.json
deleted file mode 100644
index cbbec3841..000000000
--- a/roles/network_plugin/flannel/templates/network.json
+++ /dev/null
@@ -1 +0,0 @@
-{ "Network": "{{ kube_pods_subnet }}", "SubnetLen": {{ kube_network_node_prefix }}, "Backend": { "Type": "{{ flannel_backend_type }}" } }
diff --git a/roles/uploads/defaults/main.yml b/roles/uploads/defaults/main.yml
index 0774d324c..7b5797881 100644
--- a/roles/uploads/defaults/main.yml
+++ b/roles/uploads/defaults/main.yml
@@ -5,7 +5,7 @@ local_release_dir: /tmp
kube_version: v1.4.3
etcd_version: v3.0.6
-calico_version: v0.22.0
+calico_version: v0.23.0
calico_cni_version: v1.4.2
weave_version: v1.6.1
--
GitLab