From 6602760a481ab54f6b35a8daa004bad7acf0dc4e Mon Sep 17 00:00:00 2001
From: Chad Swenson <chadswen@gmail.com>
Date: Fri, 5 Oct 2018 07:52:25 -0500
Subject: [PATCH] Support multiple local volume provisioner StorageClasses
 (#3450)

- Local Volume StorageClass configuration is now manged by `local_volume_provisioner_storage_classes`, a list of maps that specifies local storage classes with `name` `host_dir` and `mount_dir` keys per entry
- Tasks and templates updated to loop through local volume storage classes
- Previous defaults for path/class names were not changed
- Fixed an issue where a `kubernetes/preinstall` was creating directories inconsistently with the `kubernetes-apps/external_provisioner/local_volume_provisioner` task
---
 .../sample/group_vars/k8s-cluster/addons.yml     | 16 +++++++++++++---
 .../local_volume_provisioner/README.md           |  2 +-
 .../local_volume_provisioner/defaults/main.yml   |  7 ++++---
 .../local_volume_provisioner/tasks/main.yml      |  8 +++++---
 .../templates/local-volume-provisioner-cm.yml.j2 |  8 +++++---
 .../templates/local-volume-provisioner-ds.yml.j2 | 12 ++++++++----
 .../local-volume-provisioner-psp.yml.j2          |  4 +++-
 .../templates/local-volume-provisioner-sc.yml.j2 |  4 +++-
 .../node/templates/kubelet-container.j2          |  6 ++++--
 .../node/templates/kubelet.rkt.service.j2        | 16 ++++++++++------
 .../preinstall/tasks/0050-create_directories.yml | 10 +++++-----
 roles/kubespray-defaults/defaults/main.yaml      |  8 +++++---
 12 files changed, 66 insertions(+), 35 deletions(-)

diff --git a/inventory/sample/group_vars/k8s-cluster/addons.yml b/inventory/sample/group_vars/k8s-cluster/addons.yml
index 35096f17f..2e70ba3ce 100644
--- a/inventory/sample/group_vars/k8s-cluster/addons.yml
+++ b/inventory/sample/group_vars/k8s-cluster/addons.yml
@@ -14,9 +14,19 @@ registry_enabled: false
 # Local volume provisioner deployment
 local_volume_provisioner_enabled: false
 # local_volume_provisioner_namespace: kube-system
-# local_volume_provisioner_base_dir: /mnt/disks
-# local_volume_provisioner_mount_dir: /mnt/disks
-# local_volume_provisioner_storage_class: local-storage
+# local_volume_provisioner_storage_classes:
+#   - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+#     host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+#     mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
+#   - name: "local-ssd"
+#     host_dir: "/mnt/local-storage/ssd"
+#     mount_dir: "/mnt/local-storage/ssd"
+#   - name: "local-hdd"
+#     host_dir: "/mnt/local-storage/hdd"
+#     mount_dir: "/mnt/local-storage/hdd"
+#   - name: "local-shared"
+#     host_dir: "/mnt/local-storage/shared"
+#     mount_dir: "/mnt/local-storage/shared"
 
 # CephFS provisioner deployment
 cephfs_provisioner_enabled: false
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md
index 900694795..cbc468f13 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/README.md
@@ -3,7 +3,7 @@ Local Storage Provisioner
 
 The local storage provisioner is NOT a dynamic storage provisioner as you would
 expect from a cloud provider. Instead, it simply creates PersistentVolumes for
-all manually created volumes located in the directory `local_volume_provisioner_base_dir`.
+all manually created volumes located in the directories specified in the `local_volume_provisioner_storage_classes.host_dir` entries.
 The default path is /mnt/disks and the rest of this doc will use that path as
 an example.
 
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml
index 4b18546d3..2a703cbf2 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/defaults/main.yml
@@ -1,5 +1,6 @@
 ---
 local_volume_provisioner_namespace: "kube-system"
-local_volume_provisioner_base_dir: /mnt/disks
-local_volume_provisioner_mount_dir: /mnt/disks
-local_volume_provisioner_storage_class: local-storage
+local_volume_provisioner_storage_classes:
+  - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+    host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+    mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
\ No newline at end of file
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml
index 070f4c00c..ba6304166 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml
@@ -2,13 +2,15 @@
 
 - name: Local Volume Provisioner | Ensure base dir is created on all hosts
   file:
-    path: "{{ local_volume_provisioner_base_dir }}"
+    path: "{{ item[1].host_dir }}"
     state: directory
     owner: root
     group: root
     mode: 0700
-  delegate_to: "{{ item }}"
-  with_items: "{{ groups['k8s-cluster'] }}"
+  delegate_to: "{{ item[0] }}"
+  with_nested:
+    - "{{ groups['k8s-cluster'] }}"
+    - "{{ local_volume_provisioner_storage_classes }}"
   failed_when: false
 
 - name: Local Volume Provisioner | Create addon dir
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2
index 8ad76ab2d..8c5078eb6 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-cm.yml.j2
@@ -6,6 +6,8 @@ metadata:
   namespace: {{ local_volume_provisioner_namespace }}
 data:
   storageClassMap: |
-    {{ local_volume_provisioner_storage_class }}:
-      hostDir: {{ local_volume_provisioner_base_dir }}
-      mountDir: {{ local_volume_provisioner_mount_dir }}
+{% for class in local_volume_provisioner_storage_classes %}
+    {{ class.name }}:
+      hostDir: {{ class.host_dir }}
+      mountDir: {{ class.mount_dir }}
+{% endfor %}
\ No newline at end of file
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2
index 487e4f9f3..713b72287 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-ds.yml.j2
@@ -44,13 +44,17 @@ spec:
             - name: local-volume-provisioner
               mountPath: /etc/provisioner/config
               readOnly: true
-            - name: local-volume-provisioner-hostpath-mnt-disks
-              mountPath: {{ local_volume_provisioner_mount_dir }}
+{% for class in local_volume_provisioner_storage_classes %}
+            - name: {{ class.name }}
+              mountPath: {{ class.mount_dir }}
               mountPropagation: "HostToContainer"
+{% endfor %}
       volumes:
         - name: local-volume-provisioner
           configMap:
             name: local-volume-provisioner
-        - name: local-volume-provisioner-hostpath-mnt-disks
+{% for class in local_volume_provisioner_storage_classes %}
+        - name: {{ class.name }}
           hostPath:
-            path: {{ local_volume_provisioner_base_dir }}
+            path: {{ class.host_dir }}
+{% endfor %}
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2
index 9daf694fa..0358d8dda 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp.yml.j2
@@ -25,8 +25,10 @@ spec:
     - 'downwardAPI'
     - 'hostPath'
   allowedHostPaths:
-    - pathPrefix: "{{ local_volume_provisioner_base_dir }}"
+{% for class in local_volume_provisioner_storage_classes %}
+    - pathPrefix: "{{ class.host_dir }}"
       readOnly: false
+{% endfor %}
   hostNetwork: false
   hostIPC: false
   hostPID: false
diff --git a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2 b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2
index bf1f00262..2a5ad13d4 100644
--- a/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2
+++ b/roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-sc.yml.j2
@@ -1,7 +1,9 @@
+{% for class in local_volume_provisioner_storage_classes %}
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-  name: {{ local_volume_provisioner_storage_class }}
+  name: {{ class.name }}
 provisioner: kubernetes.io/no-provisioner
 volumeBindingMode: WaitForFirstConsumer
+{% endfor %}
\ No newline at end of file
diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2
index 58e54560d..c8ffbfce3 100644
--- a/roles/kubernetes/node/templates/kubelet-container.j2
+++ b/roles/kubernetes/node/templates/kubelet-container.j2
@@ -31,8 +31,10 @@
   -v {{ kubelet_flexvolumes_plugins_dir }}:{{ kubelet_flexvolumes_plugins_dir }}:rw \
   {% endif -%}
   {% if local_volume_provisioner_enabled -%}
-  -v {{ local_volume_provisioner_base_dir }}:{{ local_volume_provisioner_base_dir }}:rw \
-  -v {{ local_volume_provisioner_mount_dir }}:{{ local_volume_provisioner_mount_dir }}:rw \
+  {% for class in local_volume_provisioner_storage_classes -%}
+  -v {{ class.host_dir }}:{{ class.host_dir }}:rw \
+  -v {{ class.mount_dir }}:{{ class.mount_dir }}:rw \
+  {% endfor -%}
   {% endif %}
   -v {{kube_config_dir}}:{{kube_config_dir}}:ro \
   -v /etc/os-release:/etc/os-release:ro \
diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2
index 7f28f87b6..ee1eaa1b1 100644
--- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2
+++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2
@@ -47,11 +47,13 @@ ExecStart=/usr/bin/rkt run \
         --volume flexvolumes,kind=host,source={{ kubelet_flexvolumes_plugins_dir }},readOnly=false \
 {% endif -%}
 {% if local_volume_provisioner_enabled %}
-        --volume local-volume-provisioner-base-dir,kind=host,source={{ local_volume_provisioner_base_dir }},readOnly=false \
+{% for class in local_volume_provisioner_storage_classes %}
+        --volume local-volume-provisioner-base-dir,kind=host,source={{ class.host_dir }},readOnly=false \
 {# Not pretty, but needed to avoid double mount #}
-{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %}
-        --volume local-volume-provisioner-mount-dir,kind=host,source={{ local_volume_provisioner_mount_dir }},readOnly=false \
+{% if class.host_dir not in class.mount_dir and class.mount_dir not in class.host_dir %}
+        --volume local-volume-provisioner-mount-dir,kind=host,source={{ class.mount_dir }},readOnly=false \
 {% endif %}
+{% endfor %}
 {% endif %}
 {% if kubelet_load_modules == true %}
         --mount volume=lib-modules,target=/lib/modules \
@@ -81,11 +83,13 @@ ExecStart=/usr/bin/rkt run \
         --mount volume=flexvolumes,target={{ kubelet_flexvolumes_plugins_dir }} \
 {% endif -%}
 {% if local_volume_provisioner_enabled %}
-        --mount volume=local-volume-provisioner-base-dir,target={{ local_volume_provisioner_base_dir }} \
+{% for class in local_volume_provisioner_storage_classes %}
+        --mount volume=local-volume-provisioner-base-dir,target={{ class.host_dir }} \
 {# Not pretty, but needed to avoid double mount #}
-{% if local_volume_provisioner_base_dir not in local_volume_provisioner_mount_dir and local_volume_provisioner_mount_dir not in local_volume_provisioner_base_dir %}
-        --mount volume=local-volume-provisioner-mount-dir,target={{ local_volume_provisioner_mount_dir }} \
+{% if class.host_dir not in class.mount_dir and class.mount_dir not in class.host_dir %}
+        --mount volume=local-volume-provisioner-mount-dir,target={{ class.mount_dir }} \
 {% endif %}
+{% endfor %}
 {% endif %}
         --stage1-from-dir=stage1-fly.aci \
 {% if kube_hyperkube_image_repo == "docker" %}
diff --git a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml
index 11f8e00d4..307116033 100644
--- a/roles/kubernetes/preinstall/tasks/0050-create_directories.yml
+++ b/roles/kubernetes/preinstall/tasks/0050-create_directories.yml
@@ -46,12 +46,12 @@
 
 - name: Create local volume provisioner directories
   file:
-    path: "{{ item }}"
+    path: "{{ item.host_dir }}"
     state: directory
-    owner: kube
-  with_items:
-    - "{{ local_volume_provisioner_base_dir }}"
-    - "{{ local_volume_provisioner_mount_dir }}"
+    owner: root
+    group: root
+    mode: 0700
+  with_items: "{{ local_volume_provisioner_storage_classes }}"
   when:
     - inventory_hostname in groups['k8s-cluster']
     - local_volume_provisioner_enabled
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index ef691f10d..8ddd956c7 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -301,9 +301,11 @@ vault_config_dir: "{{ vault_base_dir }}/config"
 vault_roles_dir: "{{ vault_base_dir }}/roles"
 vault_secrets_dir: "{{ vault_base_dir }}/secrets"
 
-# Local volume provisioner dirs
-local_volume_provisioner_base_dir: /mnt/disks
-local_volume_provisioner_mount_dir: /mnt/disks
+# Local volume provisioner storage classes
+local_volume_provisioner_storage_classes:
+  - name: "{{ local_volume_provisioner_storage_class | default('local-storage') }}"
+    host_dir: "{{ local_volume_provisioner_base_dir | default ('/mnt/disks') }}"
+    mount_dir: "{{ local_volume_provisioner_mount_dir | default('/mnt/disks') }}"
 
 # weave's network password for encryption
 # if null then no network encryption
-- 
GitLab