Merge pull request #2333 from hswong3i/cephfs_provisioner_fixup

CephFS Provisioner Addon Fixup

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/README.md

+CephFS Volume Provisioner for Kubernetes 1.5+
+=============================================
+
+[![Docker Repository on Quay](https://quay.io/repository/external_storage/cephfs-provisioner/status "Docker Repository on Quay")](https://quay.io/repository/external_storage/cephfs-provisioner)
+
+Using Ceph volume client
+
+Development
+-----------
+
+Compile the provisioner
+
+``` console
+make
+```
+
+Make the container image and push to the registry
+
+``` console
+make push
+```
+
+Test instruction
+----------------
+
+-   Start Kubernetes local cluster
+
+See <a href="https://kubernetes.io/" class="uri" class="uri">https://kubernetes.io/</a>.
+
+-   Create a Ceph admin secret
+
+``` bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create ns cephfs
+kubectl create secret generic ceph-secret-admin --from-file=/tmp/secret --namespace=cephfs
+```
+
+-   Start CephFS provisioner
+
+The following example uses `cephfs-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+``` bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host  cephfs-provisioner /usr/local/bin/cephfs-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=cephfs-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+-   Create a CephFS Storage Class
+
+Replace Ceph monitor's IP in <a href="example/class.yaml" class="uri" class="uri">example/class.yaml</a> with your own and create storage class:
+
+``` bash
+kubectl create -f example/class.yaml
+```
+
+-   Create a claim
+
+``` bash
+kubectl create -f example/claim.yaml
+```
+
+-   Create a Pod using the claim
+
+``` bash
+kubectl create -f example/test-pod.yaml
+```
+
+Known limitations
+-----------------
+
+-   Kernel CephFS doesn't work with SELinux, setting SELinux label in Pod's securityContext will not work.
+-   Kernel CephFS doesn't support quota or capacity, capacity requested by PVC is not enforced or validated.
+-   Currently each Ceph user created by the provisioner has `allow r` MDS cap to permit CephFS mount.
+
+Acknowledgement
+---------------
+
+Inspired by CephFS Manila provisioner and conversation with John Spray

roles/kubernetes-apps/cephfs_provisioner/tasks/main.yml → roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml

@@ -3,22 +3,23 @@
 - name: CephFS Provisioner | Create addon dir
   file:
     path: "{{ kube_config_dir }}/addons/cephfs_provisioner"
+    state: directory
     owner: root
     group: root
     mode: 0755
-    recurse: true
 
 - name: CephFS Provisioner | Create manifests
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
   with_items:
+    - { name: cephfs-provisioner-ns, file: cephfs-provisioner-ns.yml, type: ns }
     - { name: cephfs-provisioner-sa, file: cephfs-provisioner-sa.yml, type: sa }
     - { name: cephfs-provisioner-role, file: cephfs-provisioner-role.yml, type: role }
     - { name: cephfs-provisioner-rolebinding, file: cephfs-provisioner-rolebinding.yml, type: rolebinding }
     - { name: cephfs-provisioner-clusterrole, file: cephfs-provisioner-clusterrole.yml, type: clusterrole }
     - { name: cephfs-provisioner-clusterrolebinding, file: cephfs-provisioner-clusterrolebinding.yml, type: clusterrolebinding }
-    - { name: cephfs-provisioner-deploy, file: cephfs-provisioner-deploy.yml, type: deploy }
+    - { name: cephfs-provisioner-rs, file: cephfs-provisioner-rs.yml, type: rs }
     - { name: cephfs-provisioner-secret, file: cephfs-provisioner-secret.yml, type: secret }
     - { name: cephfs-provisioner-sc, file: cephfs-provisioner-sc.yml, type: sc }
   register: cephfs_manifests
@@ -27,7 +28,7 @@
 - name: CephFS Provisioner | Apply manifests
   kube:
     name: "{{ item.item.name }}"
-    namespace: "{{ system_namespace }}"
+    namespace: "{{ cephfs_provisioner_namespace }}"
     kubectl: "{{ bin_dir }}/kubectl"
     resource: "{{ item.item.type }}"
     filename: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.item.file }}"

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2 → roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-clusterrole.yml.j2

@@ -3,7 +3,7 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: cephfs-provisioner
-  namespace: {{ system_namespace }}
+  namespace: {{ cephfs_provisioner_namespace }}
 rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-ns.yml.j2

+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ cephfs_provisioner_namespace }}
+  labels:
+    name: {{ cephfs_provisioner_namespace }}

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2 → roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rolebinding.yml.j2

@@ -7,6 +7,7 @@ metadata:
 subjects:
   - kind: ServiceAccount
     name: cephfs-provisioner
+    namespace: {{ cephfs_provisioner_namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role

roles/kubernetes-apps/cephfs_provisioner/templates/cephfs-provisioner-deploy.yml.j2 → roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/cephfs-provisioner-rs.yml.j2

 ---
-apiVersion: extensions/v1beta1
-kind: Deployment
+apiVersion: apps/v1
+kind: ReplicaSet
 metadata:
-  name: cephfs-provisioner
+  name: cephfs-provisioner-v{{ cephfs_provisioner_image_tag }}
   namespace: {{ cephfs_provisioner_namespace }}
+  labels:
+    k8s-app: cephfs-provisioner
+    version: v{{ cephfs_provisioner_image_tag }}
 spec:
   replicas: 1
-  strategy:
-    type: Recreate
+  selector:
+    matchLabels:
+      k8s-app: cephfs-provisioner
+      version: v{{ cephfs_provisioner_image_tag }}
   template:
     metadata:
       labels:
-        app: cephfs-provisioner
+        k8s-app: cephfs-provisioner
+        version: v{{ cephfs_provisioner_image_tag }}
     spec:
       containers:
         - name: cephfs-provisioner
           image: {{ cephfs_provisioner_image_repo }}:{{ cephfs_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
           env:
             - name: PROVISIONER_NAME
               value: ceph.com/cephfs
@@ -23,4 +30,6 @@ spec:
             - "/usr/local/bin/cephfs-provisioner"
           args:
             - "-id=cephfs-provisioner-1"
+{% if rbac_enabled %}
       serviceAccount: cephfs-provisioner
+{% endif %}

roles/kubernetes-apps/external_provisioner/meta/main.yml

@@ -6,3 +6,10 @@ dependencies:
       - apps
       - local-volume-provisioner
       - external-provisioner
+
+  - role: kubernetes-apps/external_provisioner/cephfs_provisioner
+    when: cephfs_provisioner_enabled
+    tags:
+      - apps
+      - cephfs-provisioner
+      - external-provisioner

roles/kubernetes-apps/meta/main.yml

@@ -27,13 +27,6 @@ dependencies:
       - apps
       - registry
 
-  - role: kubernetes-apps/cephfs_provisioner
-    when: cephfs_provisioner_enabled
-    tags:
-      - apps
-      - cephfs_provisioner
-      - storage
-
   # istio role should be last because it takes a long time to initialize and
   # will cause timeouts trying to start other addons.
   - role: kubernetes-apps/istio