add RBD Provisioner Addon (#3667) (#3668)

Based on the CephFS Provisioner Addon, the following changes have been made:
 - Upstream v2.1.1-k8s1.11
 - Configurable Provisioner replicas

README.md

@@ -124,6 +124,7 @@ Supported Components
     -   [weave](https://github.com/weaveworks/weave) v2.5.1
 -   Application
     -   [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11
+    -   [rbd-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.1-k8s1.11
     -   [cert-manager](https://github.com/jetstack/cert-manager) v0.5.2
     -   [coredns](https://github.com/coredns/coredns) v1.5.0
     -   [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.21.0

inventory/sample/group_vars/k8s-cluster/addons.yml

@@ -56,6 +56,25 @@ cephfs_provisioner_enabled: false
 # cephfs_provisioner_claim_root: /volumes
 # cephfs_provisioner_deterministic_names: true
 
+# RBD provisioner deployment
+rbd_provisioner_enabled: false
+# rbd_provisioner_namespace: rbd-provisioner
+# rbd_provisioner_replicas: 2
+# rbd_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789"
+# rbd_provisioner_pool: kube
+# rbd_provisioner_admin_id: admin
+# rbd_provisioner_secret_name: ceph-secret-admin
+# rbd_provisioner_secret: ceph-key-admin
+# rbd_provisioner_user_id: kube
+# rbd_provisioner_user_secret_name: ceph-secret-user
+# rbd_provisioner_user_secret: ceph-key-user
+# rbd_provisioner_user_secret_namespace: rbd-provisioner
+# rbd_provisioner_fs_type: ext4
+# rbd_provisioner_image_format: "2"
+# rbd_provisioner_image_features: layering
+# rbd_provisioner_storage_class: rbd
+# rbd_provisioner_reclaim_policy: Delete
+
 # Nginx ingress controller deployment
 ingress_nginx_enabled: false
 # ingress_nginx_host_network: false

roles/bootstrap-os/tasks/main.yml

@@ -59,3 +59,10 @@
     filter: ansible_hostname
   when:
     - hostname_changed.changed
+
+- name: "Install ceph-commmon package"
+  package:
+    name:
+      - ceph-common
+    state: latest
+  when: rbd_provisioner_enabled|default(false)

roles/download/defaults/main.yml

@@ -244,6 +244,8 @@ local_volume_provisioner_image_repo: "quay.io/external_storage/local-volume-prov
 local_volume_provisioner_image_tag: "v2.1.0"
 cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner"
 cephfs_provisioner_image_tag: "v2.1.0-k8s1.11"
+rbd_provisioner_image_repo: "quay.io/external_storage/rbd-provisioner"
+rbd_provisioner_image_tag: "v2.1.1-k8s1.11"
 local_path_provisioner_image_repo: "rancher/local-path-provisioner"
 local_path_provisioner_image_tag: "v0.0.2"
 ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller"
@@ -632,6 +634,15 @@ downloads:
     groups:
       - kube-node
 
+  rbd_provisioner:
+    enabled: "{{ rbd_provisioner_enabled }}"
+    container: true
+    repo: "{{ rbd_provisioner_image_repo }}"
+    tag: "{{ rbd_provisioner_image_tag }}"
+    sha256: "{{ rbd_provisioner_digest_checksum|default(None) }}"
+    groups:
+      - kube-node
+
   local_path_provisioner:
     enabled: "{{ local_volume_provisioner_enabled }}"
     container: true

roles/kubernetes-apps/external_provisioner/meta/main.yml

@@ -16,6 +16,12 @@ dependencies:
       - cephfs-provisioner
       - external-provisioner
 
+  - role: kubernetes-apps/external_provisioner/rbd_provisioner
+    when: rbd_provisioner_enabled
+    tags:
+      - apps
+      - rbd-provisioner
+      - external-provisioner
   - role: kubernetes-apps/external_provisioner/local_path_provisioner
     when: local_path_provisioner_enabled
     tags:

roles/kubernetes-apps/external_provisioner/rbd_provisioner/README.md

+# RBD Volume Provisioner for Kubernetes 1.5+
+
+`rbd-provisioner` is an out-of-tree dynamic provisioner for Kubernetes 1.5+.
+You can use it quickly & easily deploy ceph RBD storage that works almost
+anywhere.
+
+It works just like in-tree dynamic provisioner. For more information on how
+dynamic provisioning works, see [the docs](http://kubernetes.io/docs/user-guide/persistent-volumes/)
+or [this blog post](http://blog.kubernetes.io/2016/10/dynamic-provisioning-and-storage-in-kubernetes.html).
+
+## Development
+
+Compile the provisioner
+
+```console
+make
+```
+
+Make the container image and push to the registry
+
+```console
+make push
+```
+
+## Test instruction
+
+* Start Kubernetes local cluster
+
+See https://kubernetes.io/.
+
+* Create a Ceph admin secret
+
+```bash
+ceph auth get client.admin 2>&1 |grep "key = " |awk '{print  $3'} |xargs echo -n > /tmp/secret
+kubectl create secret generic ceph-admin-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Create a Ceph pool and a user secret
+
+```bash
+ceph osd pool create kube 8 8
+ceph auth add client.kube mon 'allow r' osd 'allow rwx pool=kube'
+ceph auth get-key client.kube > /tmp/secret
+kubectl create secret generic ceph-secret --from-file=/tmp/secret --namespace=kube-system
+```
+
+* Start RBD provisioner
+
+The following example uses `rbd-provisioner-1` as the identity for the instance and assumes kubeconfig is at `/root/.kube`. The identity should remain the same if the provisioner restarts. If there are multiple provisioners, each should have a different identity.
+
+```bash
+docker run -ti -v /root/.kube:/kube -v /var/run/kubernetes:/var/run/kubernetes --privileged --net=host quay.io/external_storage/rbd-provisioner /usr/local/bin/rbd-provisioner -master=http://127.0.0.1:8080 -kubeconfig=/kube/config -id=rbd-provisioner-1
+```
+
+Alternatively, deploy it in kubernetes, see [deployment](deploy/README.md).
+
+* Create a RBD Storage Class
+
+Replace Ceph monitor's IP in [examples/class.yaml](examples/class.yaml) with your own and create storage class:
+
+```bash
+kubectl create -f examples/class.yaml
+```
+
+* Create a claim
+
+```bash
+kubectl create -f examples/claim.yaml
+```
+
+* Create a Pod using the claim
+
+```bash
+kubectl create -f examples/test-pod.yaml
+```
+
+## Acknowledgements
+
+- This provisioner is extracted from [Kubernetes core](https://github.com/kubernetes/kubernetes) with some modifications for this project.

roles/kubernetes-apps/external_provisioner/rbd_provisioner/defaults/main.yml

+---
+rbd_provisioner_namespace: "rbd-provisioner"
+rbd_provisioner_replicas: 2
+rbd_provisioner_monitors: ~
+rbd_provisioner_pool: kube
+rbd_provisioner_admin_id: admin
+rbd_provisioner_secret_name: ceph-secret-admin
+rbd_provisioner_secret_token: ceph-key-admin
+rbd_provisioner_user_id: kube
+rbd_provisioner_user_secret_name: ceph-secret-user
+rbd_provisioner_user_secret_token: ceph-key-user
+rbd_provisioner_user_secret_namespace: rbd-provisioner
+rbd_provisioner_fs_type: ext4
+rbd_provisioner_image_format: "2"
+rbd_provisioner_image_features: layering
+rbd_provisioner_storage_class: rbd
+rbd_provisioner_reclaim_policy: Delete

roles/kubernetes-apps/external_provisioner/rbd_provisioner/tasks/main.yml

+---
+
+- name: RBD Provisioner | Remove legacy addon dir and manifests
+  file:
+    path: "{{ kube_config_dir }}/addons/rbd_provisioner"
+    state: absent
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+  tags:
+    - upgrade
+
+- name: RBD Provisioner | Remove legacy namespace
+  shell: |
+    {{ bin_dir }}/kubectl delete namespace {{ rbd_provisioner_namespace }}
+  ignore_errors: yes
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+  tags:
+    - upgrade
+
+- name: RBD Provisioner | Remove legacy storageclass
+  shell: |
+    {{ bin_dir }}/kubectl delete storageclass {{ rbd_provisioner_storage_class }}
+  ignore_errors: yes
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+  tags:
+    - upgrade
+
+- name: RBD Provisioner | Create addon dir
+  file:
+    path: "{{ kube_config_dir }}/addons/rbd_provisioner"
+    state: directory
+    owner: root
+    group: root
+    mode: 0755
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: RBD Provisioner | Templates list
+  set_fact:
+    rbd_provisioner_templates:
+      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
+      - { name: secret-rbd-provisioner, file: secret-rbd-provisioner.yml, type: secret }
+      - { name: sa-rbd-provisioner, file: sa-rbd-provisioner.yml, type: sa }
+      - { name: clusterrole-rbd-provisioner, file: clusterrole-rbd-provisioner.yml, type: clusterrole }
+      - { name: clusterrolebinding-rbd-provisioner, file: clusterrolebinding-rbd-provisioner.yml, type: clusterrolebinding }
+      - { name: role-rbd-provisioner, file: role-rbd-provisioner.yml, type: role }
+      - { name: rolebinding-rbd-provisioner, file: rolebinding-rbd-provisioner.yml, type: rolebinding }
+      - { name: deploy-rbd-provisioner, file: deploy-rbd-provisioner.yml, type: deploy }
+      - { name: sc-rbd-provisioner, file: sc-rbd-provisioner.yml, type: sc }
+    rbd_provisioner_templates_for_psp:
+      - { name: psp-rbd-provisioner, file: psp-rbd-provisioner.yml, type: psp }
+
+- name: RBD Provisioner | Append extra templates to RBD Provisioner Templates list for PodSecurityPolicy
+  set_fact:
+    rbd_provisioner_templates: "{{ rbd_provisioner_templates_for_psp + rbd_provisioner_templates }}"
+  when:
+    - podsecuritypolicy_enabled
+    - rbd_provisioner_namespace != "kube-system"
+
+- name: RBD Provisioner | Create manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.file }}"
+  with_items: "{{ rbd_provisioner_templates }}"
+  register: rbd_provisioner_manifests
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: RBD Provisioner | Apply manifests
+  kube:
+    name: "{{ item.item.name }}"
+    namespace: "{{ rbd_provisioner_namespace }}"
+    kubectl: "{{ bin_dir }}/kubectl"
+    resource: "{{ item.item.type }}"
+    filename: "{{ kube_config_dir }}/addons/rbd_provisioner/{{ item.item.file }}"
+    state: "latest"
+  with_items: "{{ rbd_provisioner_manifests.results }}"
+  when: inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/00-namespace.yml.j2

+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ rbd_provisioner_namespace }}
+  labels:
+    name: {{ rbd_provisioner_namespace }}

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrole-rbd-provisioner.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["services"]
+    resourceNames: ["kube-dns","coredns"]
+    verbs: ["list", "get"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "delete"]
+  - apiGroups: ["policy"]
+    resourceNames: ["rbd-provisioner"]
+    resources: ["podsecuritypolicies"]
+    verbs: ["use"]

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/clusterrolebinding-rbd-provisioner.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: rbd-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: rbd-provisioner
+    namespace: {{ rbd_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: rbd-provisioner
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/deploy-rbd-provisioner.yml.j2

+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+  labels:
+    app: rbd-provisioner
+    version: {{ rbd_provisioner_image_tag }}
+spec:
+  replicas: {{ rbd_provisioner_replicas }}
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: rbd-provisioner
+      version: {{ rbd_provisioner_image_tag }}
+  template:
+    metadata:
+      labels:
+        app: rbd-provisioner
+        version: {{ rbd_provisioner_image_tag }}
+    spec:
+{% if kube_version is version('v1.11.1', '>=') %}
+      priorityClassName: {% if rbd_provisioner_namespace == 'kube-system' %}system-cluster-critical{% else %}k8s-cluster-critical{% endif %}{{''}}
+{% endif %}
+      serviceAccount: rbd-provisioner
+      containers:
+        - name: rbd-provisioner
+          image: {{ rbd_provisioner_image_repo }}:{{ rbd_provisioner_image_tag }}
+          imagePullPolicy: {{ k8s_image_pull_policy }}
+          env:
+            - name: PROVISIONER_NAME
+              value: ceph.com/rbd
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          command:
+            - "/usr/local/bin/rbd-provisioner"
+          args:
+            - "-id=${POD_NAME}"

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/psp-rbd-provisioner.yml.j2

+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: rbd-provisioner
+  annotations:
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+{% if apparmor_enabled %}
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+{% endif %}
+  labels:
+    kubernetes.io/cluster-service: 'true'
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/role-rbd-provisioner.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/rolebinding-rbd-provisioner.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: rbd-provisioner
+    namespace: {{ rbd_provisioner_namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: rbd-provisioner

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sa-rbd-provisioner.yml.j2

+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: rbd-provisioner
+  namespace: {{ rbd_provisioner_namespace }}

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/sc-rbd-provisioner.yml.j2

+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: {{ rbd_provisioner_storage_class }}
+provisioner: ceph.com/rbd
+reclaimPolicy: {{ rbd_provisioner_reclaim_policy }}
+parameters:
+  monitors: {{ rbd_provisioner_monitors }}
+  adminId: {{ rbd_provisioner_admin_id }}
+  adminSecretNamespace: {{ rbd_provisioner_namespace }}
+  adminSecretName: {{ rbd_provisioner_secret_name }}
+  pool: {{ rbd_provisioner_pool }}
+  userId: {{ rbd_provisioner_user_id }}
+  userSecretNamespace: {{ rbd_provisioner_user_secret_namespace }}
+  userSecretName: {{ rbd_provisioner_user_secret_name }}
+  fsType: "{{ rbd_provisioner_fs_type }}"
+  imageFormat: "{{ rbd_provisioner_image_format }}"
+  imageFeatures: {{ rbd_provisioner_image_features }}

roles/kubernetes-apps/external_provisioner/rbd_provisioner/templates/secret-rbd-provisioner.yml.j2

+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ rbd_provisioner_secret_name }}
+  namespace: {{ rbd_provisioner_namespace }}
+type: Opaque
+data:
+  secret: {{ rbd_provisioner_secret | b64encode }}
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: {{ rbd_provisioner_user_secret_name }}
+  namespace: {{ rbd_provisioner_user_secret_namespace }}
+type: Opaque
+data:
+  key: {{ rbd_provisioner_user_secret | b64encode }}

roles/kubespray-defaults/defaults/main.yaml

@@ -278,6 +278,7 @@ enable_network_policy: true
 local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}"
 persistent_volumes_enabled: false
 cephfs_provisioner_enabled: false
+rbd_provisioner_enabled: false
 ingress_nginx_enabled: false
 cert_manager_enabled: false
 expand_persistent_volumes: false