From 6ffddbff24cec4cfc42def0b466f504aac8f5610 Mon Sep 17 00:00:00 2001
From: Sascha Marcel Schmidt <mail@saschaschmidt.net>
Date: Wed, 5 Sep 2018 16:03:32 +0200
Subject: [PATCH] fix database not available heketi error
---
.../roles/provision/tasks/bootstrap.yml | 11 ++++-----
.../provision/tasks/bootstrap/deploy.yml | 1 +
.../provision/tasks/bootstrap/storage.yml | 23 +++++++++++--------
.../provision/tasks/bootstrap/topology.yml | 6 ++++-
.../provision/tasks/bootstrap/volumes.yml | 2 +-
.../roles/provision/tasks/glusterfs.yml | 15 ++++++------
.../heketi/roles/provision/tasks/heketi.yml | 1 +
.../heketi/roles/provision/tasks/main.yml | 3 +++
.../heketi/roles/provision/tasks/topology.yml | 2 +-
.../templates/glusterfs-daemonset.json.j2 | 2 +-
10 files changed, 39 insertions(+), 27 deletions(-)
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml
index 788f01f4b..572913a63 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml
@@ -32,13 +32,16 @@
- name: "Prepare heketi volumes."
include_tasks: "bootstrap/volumes.yml"
+# Remove bootstrap heketi
+- name: "Tear down bootstrap."
+ include_tasks: "bootstrap/tear-down.yml"
+
# Prepare heketi storage
- name: "Test heketi storage."
command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
changed_when: false
register: "heketi_storage_state"
-- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
- register: "job"
+# ensure endpoints actually exist before trying to move database data to it
- name: "Create heketi storage."
include_tasks: "bootstrap/storage.yml"
vars:
@@ -51,7 +54,3 @@
- "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
-
-# Remove bootstrap heketi
-- name: "Tear down bootstrap."
- include_tasks: "bootstrap/tear-down.yml"
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
index 3580707d5..3037d8b77 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml
@@ -6,6 +6,7 @@
- name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
kube:
name: "GlusterFS"
+ kubectl: "{{bin_dir}}/kubectl"
filename: "{{ kube_config_dir }}/heketi-bootstrap.json"
state: "{{ rendering.changed | ternary('latest', 'present') }}"
- name: "Wait for heketi bootstrap to complete."
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml
index 1daa72ac1..ff32ef201 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml
@@ -6,7 +6,8 @@
- name: "Create heketi storage."
kube:
name: "GlusterFS"
- filename: "{{ kube_config_dir }}/heketi-storage.json"
+ kubectl: "{{bin_dir}}/kubectl"
+ filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json"
state: "present"
vars:
secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
@@ -19,21 +20,23 @@
- "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
register: "heketi_storage_result"
-
-- name: "Get state of heketi storage service, endpoint, secret and job."
- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+- name: "Get state of heketi database copy job."
+ command: "{{ bin_dir }}/kubectl get jobs --output=json"
changed_when: false
register: "heketi_storage_state"
vars:
heketi_storage_state: { stdout: "{}" }
- secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
- endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
- service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]"
until:
- - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
- - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
- - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0"
- "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1"
retries: 60
delay: 5
+- become: true
+ template: { src: "heketi-storage-test.json.j2", dest: "{{ kube_config_dir }}/heketi-storage-test.json" }
+ register: "rendering"
+- kube:
+ name: "GlusterFS"
+ kubectl: "{{bin_dir}}/kubectl"
+ filename: "{{ kube_config_dir }}/heketi-storage-test.json"
+ state: "{{ rendering.changed | ternary('latest', 'present') }}"
+ register: "state"
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
index 31d1d48cb..7d2c5981e 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml
@@ -1,20 +1,24 @@
---
- name: "Get heketi topology."
+ changed_when: false
register: "heketi_topology"
command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
- name: "Render heketi topology template."
become: true
vars: { nodes: "{{ groups['heketi-node'] }}" }
+ register: "render"
template:
src: "topology.json.j2"
dest: "{{ kube_config_dir }}/topology.json"
- name: "Copy topology configuration into container."
+ changed_when: false
command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
- name: "Load heketi topology."
- when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+ when: "render.changed"
command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
register: "load_heketi"
- name: "Get heketi topology."
+ changed_when: false
register: "heketi_topology"
command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml
index bb088173e..d5da1a125 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml
@@ -20,7 +20,7 @@
when: "heketi_database_volume_exists is undefined"
- name: "Copy configuration from pod."
become: true
- command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json"
+ command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json"
- name: "Get heketi volume ids."
command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
changed_when: false
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
index e9650276a..e46159969 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml
@@ -6,8 +6,14 @@
- name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
kube:
name: "GlusterFS"
+ kubectl: "{{bin_dir}}/kubectl"
filename: "{{ kube_config_dir }}/glusterfs-daemonset.json"
state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Kubernetes Apps | Label GlusterFS nodes"
+ include_tasks: "glusterfs/label.yml"
+ with_items: "{{ groups['heketi-node'] }}"
+ loop_control:
+ loop_var: "node"
- name: "Kubernetes Apps | Wait for daemonset to become available."
register: "daemonset_state"
command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
@@ -16,16 +22,10 @@
daemonset_state: { stdout: "{}" }
ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}"
desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}"
- until: "ready == desired"
+ until: "ready >= 3"
retries: 60
delay: 5
-- name: "Kubernetes Apps | Label GlusterFS nodes"
- include_tasks: "glusterfs/label.yml"
- with_items: "{{ groups['heketi-node'] }}"
- loop_control:
- loop_var: "node"
-
- name: "Kubernetes Apps | Lay Down Heketi Service Account"
template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
become: true
@@ -33,5 +33,6 @@
- name: "Kubernetes Apps | Install and configure Heketi Service Account"
kube:
name: "GlusterFS"
+ kubectl: "{{bin_dir}}/kubectl"
filename: "{{ kube_config_dir }}/heketi-service-account.json"
state: "{{ rendering.changed | ternary('latest', 'present') }}"
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
index 44016df5e..029baef94 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/heketi.yml
@@ -6,6 +6,7 @@
- name: "Kubernetes Apps | Install and configure Heketi"
kube:
name: "GlusterFS"
+ kubectl: "{{bin_dir}}/kubectl"
filename: "{{ kube_config_dir }}/heketi-deployment.json"
state: "{{ rendering.changed | ternary('latest', 'present') }}"
- name: "Ensure heketi is up and running."
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/main.yml b/contrib/network-storage/heketi/roles/provision/tasks/main.yml
index 4adc2dbb2..23a2b4f9c 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/main.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/main.yml
@@ -25,3 +25,6 @@
- name: "Kubernetes Apps | Storage Class"
include_tasks: "storageclass.yml"
+
+- name: "Clean up"
+ include_tasks: "cleanup.yml"
diff --git a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml
index b9cb3e0d2..dd1e272be 100644
--- a/contrib/network-storage/heketi/roles/provision/tasks/topology.yml
+++ b/contrib/network-storage/heketi/roles/provision/tasks/topology.yml
@@ -2,7 +2,7 @@
- name: "Get heketi topology."
register: "heketi_topology"
changed_when: false
- command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json"
+ command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
- name: "Render heketi topology template."
become: true
vars: { nodes: "{{ groups['heketi-node'] }}" }
diff --git a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2 b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2
index 8f46bea24..eddd57eb8 100644
--- a/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2
+++ b/contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2
@@ -27,7 +27,7 @@
"containers": [
{
"image": "gluster/gluster-centos:gluster4u0_centos7",
- "imagePullPolicy": "Always",
+ "imagePullPolicy": "IfNotPresent",
"name": "glusterfs",
"volumeMounts": [
{
--
GitLab