From df28db00662547eee247c0b21923d2f2fe2bcdcf Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <mmosesohn@mirantis.com>
Date: Fri, 18 Aug 2017 15:46:22 +0300
Subject: [PATCH] Fix cert and netchecker upgrade issues (#1543)
* Bump tag for upgrade CI, fix netchecker upgrade
netchecker-server was changed from pod to deployment, so
we need an upgrade hook for it.
CI now uses v2.1.1 as a basis for upgrade.
* Fix upgrades for certs from non-rbac to rbac
---
.gitlab-ci.yml | 4 +-
.../ansible/tasks/netchecker.yml | 9 ++++
roles/kubernetes/secrets/files/make-ssl.sh | 7 ++-
.../kubernetes/secrets/tasks/check-certs.yml | 50 +++++++++++++++++--
.../secrets/tasks/gen_certs_script.yml | 2 +-
5 files changed, 60 insertions(+), 12 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b7f3f7f4a..a7e5130c8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -110,7 +110,7 @@ before_script:
# Check out latest tag if testing upgrade
# Uncomment when gitlab kargo repo has tags
#- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1))
- - test "${UPGRADE_TEST}" != "false" && git checkout 031cf565ec3ccd3ebbe80eeef3454c3780e5c598 && pip install ansible==2.2.0
+ - test "${UPGRADE_TEST}" != "false" && git checkout acae0fe4a36bd1d3cd267e72ad01126a72d1458a
# Create cluster
@@ -142,7 +142,6 @@ before_script:
if [ "${UPGRADE_TEST}" != "false" ]; then
test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml";
test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml";
- pip install ansible==2.3.0;
git checkout "${CI_BUILD_REF}";
ansible-playbook -i inventory/inventory.ini -b --become-user=root --private-key=${HOME}/.ssh/id_rsa -u $SSH_USER
${SSH_ARGS}
@@ -281,7 +280,6 @@ before_script:
CLOUD_IMAGE: ubuntu-1604-xenial
CLOUD_REGION: europe-west1-b
CLOUD_MACHINE_TYPE: "n1-standard-2"
- UPGRADE_TEST: "basic"
CLUSTER_MODE: ha
UPGRADE_TEST: "graceful"
STARTUP_SCRIPT: ""
diff --git a/roles/kubernetes-apps/ansible/tasks/netchecker.yml b/roles/kubernetes-apps/ansible/tasks/netchecker.yml
index a06290578..2d88b288c 100644
--- a/roles/kubernetes-apps/ansible/tasks/netchecker.yml
+++ b/roles/kubernetes-apps/ansible/tasks/netchecker.yml
@@ -15,6 +15,15 @@
- inventory_hostname == groups['kube-master'][0]
- rbac_enabled or item.type not in rbac_resources
+- name: Kubernetes Apps | Purge old Netchecker server
+ kube:
+ name: "netchecker-server"
+ namespace: "{{ netcheck_namespace }}"
+ kubectl: "{{bin_dir}}/kubectl"
+ resource: "po"
+ state: absent
+ when: inventory_hostname == groups['kube-master'][0]
+
#FIXME: remove if kubernetes/features#124 is implemented
- name: Kubernetes Apps | Purge old Netchecker daemonsets
kube:
diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh
index e8574cc6b..0cb7e37c6 100755
--- a/roles/kubernetes/secrets/files/make-ssl.sh
+++ b/roles/kubernetes/secrets/files/make-ssl.sh
@@ -80,7 +80,9 @@ gen_key_and_cert() {
openssl x509 -req -in ${name}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${name}.pem -days 3650 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
}
-if [ ! -e "$SSLDIR/ca-key.pem" ]; then
+# Admins
+if [ -n "$MASTERS" ]; then
+ # If any host requires new certs, just regenerate all master certs
# kube-apiserver
gen_key_and_cert "apiserver" "/CN=kube-apiserver"
cat ca.pem >> apiserver.pem
@@ -88,10 +90,7 @@ if [ ! -e "$SSLDIR/ca-key.pem" ]; then
gen_key_and_cert "kube-scheduler" "/CN=system:kube-scheduler"
# kube-controller-manager
gen_key_and_cert "kube-controller-manager" "/CN=system:kube-controller-manager"
-fi
-# Admins
-if [ -n "$MASTERS" ]; then
for host in $MASTERS; do
cn="${host%%.*}"
# admin
diff --git a/roles/kubernetes/secrets/tasks/check-certs.yml b/roles/kubernetes/secrets/tasks/check-certs.yml
index 061b04b04..69b82d957 100644
--- a/roles/kubernetes/secrets/tasks/check-certs.yml
+++ b/roles/kubernetes/secrets/tasks/check-certs.yml
@@ -20,7 +20,18 @@
register: kubecert_node
with_items:
- ca.pem
+ - apiserver.pem
+ - apiserver-key.pem
+ - kube-scheduler.pem
+ - kube-scheduler-key.pem
+ - kube-controller-manager.pem
+ - kube-controller-manager-key.pem
+ - admin-{{ inventory_hostname }}.pem
+ - admin-{{ inventory_hostname }}-key.pem
+ - node-{{ inventory_hostname }}.pem
- node-{{ inventory_hostname }}-key.pem
+ - kube-proxy-{{ inventory_hostname }}.pem
+ - kube-proxy-{{ inventory_hostname }}-key.pem
- name: "Check_certs | Set 'gen_certs' to true"
set_fact:
@@ -29,11 +40,42 @@
run_once: true
with_items: >-
['{{ kube_cert_dir }}/ca.pem',
+ '{{ kube_cert_dir }}/apiserver.pem',
+ '{{ kube_cert_dir }}/apiserver-key.pem',
+ '{{ kube_cert_dir }}/kube-scheduler.pem',
+ '{{ kube_cert_dir }}/kube-scheduler-key.pem',
+ '{{ kube_cert_dir }}/kube-controller-manager.pem',
+ '{{ kube_cert_dir }}/kube-controller-manager-key.pem',
+ {% for host in groups['kube-master'] %}
+ '{{ kube_cert_dir }}/admin-{{ host }}.pem'
+ '{{ kube_cert_dir }}/admin-{{ host }}-key.pem'
+ {% if not loop.last %}{{','}}{% endif %}
+ {% endfor %}]
{% for host in groups['k8s-cluster'] %}
+ '{{ kube_cert_dir }}/node-{{ host }}.pem'
'{{ kube_cert_dir }}/node-{{ host }}-key.pem'
+ '{{ kube_cert_dir }}/kube-proxy-{{ host }}.pem'
+ '{{ kube_cert_dir }}/kube-proxy-{{ host }}-key.pem'
{% if not loop.last %}{{','}}{% endif %}
{% endfor %}]
+- name: "Check_certs | Set 'gen_master_certs' to true"
+ set_fact:
+ gen_master_certs: |-
+ {%- set gen = False -%}
+ {% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
+ {% for cert in ['apiserver.pem', 'apiserver-key.pem', 'kube-scheduler.pem',
+ 'kube-scheduler-key.pem', 'kube-controller-manager.pem',
+ 'kube-controller-manager-key.pem'] -%}
+ {% set cert_file = "%s/%s.pem"|format(kube_cert_dir, cert) %}
+ {% if not cert_file in existing_certs -%}
+ {%- set gen = True -%}
+ {% endif -%}
+ {% endfor %}
+ {{ gen }}
+ run_once: true
+
+
- name: "Check_certs | Set 'gen_node_certs' to true"
set_fact:
gen_node_certs: |-
@@ -41,7 +83,8 @@
{% set existing_certs = kubecert_master.files|map(attribute='path')|list|sort %}
{% for host in groups['k8s-cluster'] -%}
{% set host_cert = "%s/node-%s-key.pem"|format(kube_cert_dir, host) %}
- {% if host_cert in existing_certs -%}
+ {% set kube_proxy_cert = "%s/kube-proxy-%s-key.pem"|format(kube_cert_dir, host) %}
+ {% if host_cert in existing_certs and kube_proxy_cert in existing_certs -%}
"{{ host }}": False,
{% else -%}
"{{ host }}": True,
@@ -50,7 +93,6 @@
}
run_once: true
-
- name: "Check_certs | Set 'sync_certs' to true"
set_fact:
sync_certs: true
@@ -58,8 +100,8 @@
{%- set certs = {'sync': False} -%}
{% if gen_node_certs[inventory_hostname] or
(not kubecert_node.results[0].stat.exists|default(False)) or
- (not kubecert_node.results[1].stat.exists|default(False)) or
- (kubecert_node.results[1].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[1].stat.path)|map(attribute="checksum")|first|default('')) -%}
+ (not kubecert_node.results[10].stat.exists|default(False)) or
+ (kubecert_node.results[10].stat.checksum|default('') != kubecert_master.files|selectattr("path", "equalto", kubecert_node.results[10].stat.path)|map(attribute="checksum")|first|default('')) -%}
{%- set _ = certs.update({'sync': True}) -%}
{% endif %}
{{ certs.sync }}
diff --git a/roles/kubernetes/secrets/tasks/gen_certs_script.yml b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
index 61d9c7826..80fb4a506 100644
--- a/roles/kubernetes/secrets/tasks/gen_certs_script.yml
+++ b/roles/kubernetes/secrets/tasks/gen_certs_script.yml
@@ -40,7 +40,7 @@
command: "{{ kube_script_dir }}/make-ssl.sh -f {{ kube_config_dir }}/openssl.conf -d {{ kube_cert_dir }}"
environment:
- MASTERS: "{% for m in groups['kube-master'] %}
- {% if gen_node_certs[m]|default(false) %}
+ {% if gen_master_certs|default(false) %}
{{ m }}
{% endif %}
{% endfor %}"
--
GitLab