fix database not available heketi error

contrib/network-storage/heketi/roles/provision/tasks/bootstrap.yml

@@ -32,13 +32,16 @@
 - name: "Prepare heketi volumes."
   include_tasks: "bootstrap/volumes.yml"
 
+# Remove bootstrap heketi
+- name: "Tear down bootstrap."
+  include_tasks: "bootstrap/tear-down.yml"
+
 # Prepare heketi storage
 - name: "Test heketi storage."
   command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
   changed_when: false
   register: "heketi_storage_state"
-- command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
-  register: "job"
+# ensure endpoints actually exist before trying to move database data to it
 - name: "Create heketi storage."
   include_tasks: "bootstrap/storage.yml"
   vars:
@@ -51,7 +54,3 @@
     - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
-
-# Remove bootstrap heketi
-- name: "Tear down bootstrap."
-  include_tasks: "bootstrap/tear-down.yml"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/deploy.yml

@@ -6,6 +6,7 @@
 - name: "Kubernetes Apps | Install and configure Heketi Bootstrap"
   kube:
     name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
     filename: "{{ kube_config_dir }}/heketi-bootstrap.json"
     state: "{{ rendering.changed | ternary('latest', 'present') }}"
 - name: "Wait for heketi bootstrap to complete."

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/storage.yml

@@ -6,7 +6,8 @@
 - name: "Create heketi storage."
   kube:
     name: "GlusterFS"
-    filename: "{{ kube_config_dir }}/heketi-storage.json"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage-bootstrap.json"
     state: "present"
   vars:
     secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
@@ -19,21 +20,23 @@
     - "heketi_storage_state.stdout|from_json|json_query(service_query)|length == 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 0"
   register: "heketi_storage_result"
-
-- name: "Get state of heketi storage service, endpoint, secret and job."
-  command: "{{ bin_dir }}/kubectl get secrets,endpoints,services,jobs --output=json"
+- name: "Get state of heketi database copy job."
+  command: "{{ bin_dir }}/kubectl get jobs --output=json"
   changed_when: false
   register: "heketi_storage_state"
   vars:
     heketi_storage_state: { stdout: "{}" }
-    secret_query: "items[?metadata.name=='heketi-storage-secret' && kind=='Secret']"
-    endpoints_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Endpoints']"
-    service_query: "items[?metadata.name=='heketi-storage-endpoints' && kind=='Service']"
     job_query: "items[?metadata.name=='heketi-storage-copy-job' && kind=='Job' && status.succeeded==1]"
   until:
-    - "heketi_storage_state.stdout|from_json|json_query(secret_query)|length == 1"
-    - "heketi_storage_state.stdout|from_json|json_query(endpoints_query)|length == 1"
-    - "heketi_storage_state.stdout|from_json|json_query(service_query)|length > 0"
     - "heketi_storage_state.stdout|from_json|json_query(job_query)|length == 1"
   retries: 60
   delay: 5
+- become: true
+  template: { src: "heketi-storage-test.json.j2", dest: "{{ kube_config_dir }}/heketi-storage-test.json" }
+  register: "rendering"
+- kube:
+    name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
+    filename: "{{ kube_config_dir }}/heketi-storage-test.json"
+    state: "{{ rendering.changed | ternary('latest', 'present') }}"
+  register: "state"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/topology.yml

 ---
 - name: "Get heketi topology."
+  changed_when: false
   register: "heketi_topology"
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
 - name: "Render heketi topology template."
   become: true
   vars: { nodes: "{{ groups['heketi-node'] }}" }
+  register: "render"
   template:
     src: "topology.json.j2"
     dest: "{{ kube_config_dir }}/topology.json"
 - name: "Copy topology configuration into container."
+  changed_when: false
   command: "{{ bin_dir }}/kubectl cp {{ kube_config_dir }}/topology.json {{ initial_heketi_pod_name }}:/tmp/topology.json"
 - name: "Load heketi topology."
-  when: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*]\")|flatten|length == 0"
+  when: "render.changed"
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology load --json=/tmp/topology.json"
   register: "load_heketi"
 - name: "Get heketi topology."
+  changed_when: false
   register: "heketi_topology"
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
   until: "heketi_topology.stdout|from_json|json_query(\"clusters[*].nodes[*].devices[?state=='online'].id\")|flatten|length == groups['heketi-node']|length"

contrib/network-storage/heketi/roles/provision/tasks/bootstrap/volumes.yml

@@ -20,7 +20,7 @@
   when: "heketi_database_volume_exists is undefined"
 - name: "Copy configuration from pod."
   become: true
-  command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage.json"
+  command: "{{ bin_dir }}/kubectl cp {{ initial_heketi_pod_name }}:/heketi-storage.json {{ kube_config_dir }}/heketi-storage-bootstrap.json"
 - name: "Get heketi volume ids."
   command: "{{ bin_dir }}/kubectl exec {{ initial_heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} volume list --json"
   changed_when: false

contrib/network-storage/heketi/roles/provision/tasks/glusterfs.yml

@@ -6,8 +6,14 @@
 - name: "Kubernetes Apps | Install and configure GlusterFS daemonset"
   kube:
     name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
     filename: "{{ kube_config_dir }}/glusterfs-daemonset.json"
     state: "{{ rendering.changed | ternary('latest', 'present') }}"
+- name: "Kubernetes Apps | Label GlusterFS nodes"
+  include_tasks: "glusterfs/label.yml"
+  with_items: "{{ groups['heketi-node'] }}"
+  loop_control:
+    loop_var: "node"
 - name: "Kubernetes Apps | Wait for daemonset to become available."
   register: "daemonset_state"
   command: "{{ bin_dir }}/kubectl get daemonset glusterfs --output=json --ignore-not-found=true"
@@ -16,16 +22,10 @@
     daemonset_state: { stdout: "{}" }
     ready: "{{ daemonset_state.stdout|from_json|json_query(\"status.numberReady\") }}"
     desired: "{{ daemonset_state.stdout|from_json|json_query(\"status.desiredNumberScheduled\") }}"
-  until: "ready == desired"
+  until: "ready >= 3"
   retries: 60
   delay: 5
 
-- name: "Kubernetes Apps | Label GlusterFS nodes"
-  include_tasks: "glusterfs/label.yml"
-  with_items: "{{ groups['heketi-node'] }}"
-  loop_control:
-    loop_var: "node"
-
 - name: "Kubernetes Apps | Lay Down Heketi Service Account"
   template: { src: "heketi-service-account.json.j2", dest: "{{ kube_config_dir }}/heketi-service-account.json" }
   become: true
@@ -33,5 +33,6 @@
 - name: "Kubernetes Apps | Install and configure Heketi Service Account"
   kube:
     name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
     filename: "{{ kube_config_dir }}/heketi-service-account.json"
     state: "{{ rendering.changed | ternary('latest', 'present') }}"

contrib/network-storage/heketi/roles/provision/tasks/heketi.yml

@@ -6,6 +6,7 @@
 - name: "Kubernetes Apps | Install and configure Heketi"
   kube:
     name: "GlusterFS"
+    kubectl: "{{bin_dir}}/kubectl"
     filename: "{{ kube_config_dir }}/heketi-deployment.json"
     state: "{{ rendering.changed | ternary('latest', 'present') }}"
 - name: "Ensure heketi is up and running."

contrib/network-storage/heketi/roles/provision/tasks/main.yml

@@ -25,3 +25,6 @@
 
 - name: "Kubernetes Apps | Storage Class"
   include_tasks: "storageclass.yml"
+
+- name: "Clean up"
+  include_tasks: "cleanup.yml"

contrib/network-storage/heketi/roles/provision/tasks/topology.yml

@@ -2,7 +2,7 @@
 - name: "Get heketi topology."
   register: "heketi_topology"
   changed_when: false
-  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli topology info --json"
+  command: "{{ bin_dir }}/kubectl exec {{ heketi_pod_name }} -- heketi-cli --user admin --secret {{ heketi_admin_key }} topology info --json"
 - name: "Render heketi topology template."
   become: true
   vars: { nodes: "{{ groups['heketi-node'] }}" }

contrib/network-storage/heketi/roles/provision/templates/glusterfs-daemonset.json.j2

@@ -27,7 +27,7 @@
                 "containers": [
                     {
                         "image": "gluster/gluster-centos:gluster4u0_centos7",
-                        "imagePullPolicy": "Always",
+                        "imagePullPolicy": "IfNotPresent",
                         "name": "glusterfs",
                         "volumeMounts": [
                             {