From 937adec51529ab644f35640da74781d144e242e5 Mon Sep 17 00:00:00 2001
From: Ali Sanhaji <ali.sanhaji@wearelog.fr>
Date: Wed, 1 Apr 2020 09:53:27 +0200
Subject: [PATCH] Azure Disk CSI deployment (#5833)
* Azure Disk CSI deployment
* Mention Azure CSI support
* Fix: remove unnecessary file
* Typo in documentation
* Add newline to end of file
---
docs/azure-csi.md | 119 ++++++++++
inventory/sample/group_vars/all/azure.yml | 20 ++
.../group_vars/k8s-cluster/k8s-cluster.yml | 3 +-
roles/download/defaults/main.yml | 11 +
.../csi_driver/azuredisk/defaults/main.yml | 4 +
.../tasks/azure-credential-check.yml | 54 +++++
.../csi_driver/azuredisk/tasks/main.yml | 48 ++++
...azure-csi-azuredisk-controller-rbac.yml.j2 | 212 ++++++++++++++++++
.../azure-csi-azuredisk-controller.yml.j2 | 200 +++++++++++++++++
.../azure-csi-azuredisk-driver.yml.j2 | 10 +
.../templates/azure-csi-azuredisk-node.yml.j2 | 156 +++++++++++++
.../azure-csi-cloud-config-secret.yml.j2 | 7 +
.../templates/azure-csi-cloud-config.j2 | 14 ++
.../templates/azure-csi-node-info-crd.yml.j2 | 38 ++++
roles/kubernetes-apps/meta/main.yml | 8 +
.../azuredisk-csi/defaults/main.yml | 3 +
.../azuredisk-csi/tasks/main.yml | 19 ++
.../templates/azure-csi-storage-class.yml.j2 | 11 +
.../persistent_volumes/meta/main.yml | 7 +
roles/kubespray-defaults/defaults/main.yaml | 1 +
20 files changed, 944 insertions(+), 1 deletion(-)
create mode 100644 docs/azure-csi.md
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2
create mode 100644 roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2
create mode 100644 roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml
create mode 100644 roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml
create mode 100644 roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2
diff --git a/docs/azure-csi.md b/docs/azure-csi.md
new file mode 100644
index 000000000..e121c8874
--- /dev/null
+++ b/docs/azure-csi.md
@@ -0,0 +1,119 @@
+# Azure Disk CSI Driver
+
+The Azure Disk CSI driver allows you to provision volumes for pods with a Kubernetes deployment over Azure Cloud. The CSI driver replaces to volume provioning done by the in-tree azure cloud provider which is deprecated.
+
+This documentation is an updated version of the in-tree Azure cloud provider documentation (azure.md).
+
+To deploy Azure Disk CSI driver, uncomment the `azure_csi_enabled` option in `group_vars/all/azure.yml` and set it to `true`.
+
+## Azure Disk CSI Storage Class
+
+If you want to deploy the Azure Disk storage class to provision volumes dynamically, you should set `persistent_volumes_enabled` in `group_vars/k8s-cluster/k8s-cluster.yml` to `true`.
+
+## Parameters
+
+Before creating the instances you must first set the `azure_csi_` variables in the `group_vars/all.yml` file.
+
+All of the values can be retrieved using the azure cli tool which can be downloaded here: <https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest>
+
+After installation you have to run `az login` to get access to your account.
+
+### azure\_csi\_tenant\_id + azure\_csi\_subscription\_id
+
+Run `az account show` to retrieve your subscription id and tenant id:
+`azure_csi_tenant_id` -> tenantId field
+`azure_csi_subscription_id` -> id field
+
+### azure\_csi\_location
+
+The region your instances are located in, it can be something like `francecentral` or `norwayeast`. A full list of region names can be retrieved via `az account list-locations`
+
+### azure\_csi\_resource\_group
+
+The name of the resource group your instances are in, a list of your resource groups can be retrieved via `az group list`
+
+Or you can do `az vm list | grep resourceGroup` and get the resource group corresponding to the VMs of your cluster.
+
+The resource group name is not case sensitive.
+
+### azure\_csi\_vnet\_name
+
+The name of the virtual network your instances are in, can be retrieved via `az network vnet list`
+
+### azure\_csi\_vnet\_resource\_group
+
+The name of the resource group your vnet is in, can be retrieved via `az network vnet list | grep resourceGroup` and get the resource group corresponding to the vnet of your cluster.
+
+### azure\_csi\_subnet\_name
+
+The name of the subnet your instances are in, can be retrieved via `az network vnet subnet list --resource-group RESOURCE_GROUP --vnet-name VNET_NAME`
+
+### azure\_csi\_security\_group\_name
+
+The name of the network security group your instances are in, can be retrieved via `az network nsg list`
+
+### azure\_csi\_aad\_client\_id + azure\_csi\_aad\_client\_secret
+
+These will have to be generated first:
+
+- Create an Azure AD Application with:
+`az ad app create --display-name kubespray --identifier-uris http://kubespray --homepage http://kubespray.com --password CLIENT_SECRET`
+
+Display name, identifier-uri, homepage and the password can be chosen
+
+Note the AppId in the output.
+
+- Create Service principal for the application with:
+`az ad sp create --id AppId`
+
+This is the AppId from the last command
+
+- Create the role assignment with:
+`az role assignment create --role "Owner" --assignee http://kubespray --subscription SUBSCRIPTION_ID`
+
+azure\_csi\_aad\_client\_id must be set to the AppId, azure\_csi\_aad\_client\_secret is your chosen secret.
+
+### azure\_csi\_use\_instance\_metadata
+
+Use instance metadata service where possible. Boolean value.
+
+## Test the Azure Disk CSI driver
+
+To test the dynamic provisioning using Azure CSI driver, make sure to have the storage class deployed (through persistent volumes), and apply the following manifest:
+
+```yml
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: pvc-azuredisk
+spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 1Gi
+ storageClassName: disk.csi.azure.com
+---
+kind: Pod
+apiVersion: v1
+metadata:
+ name: nginx-azuredisk
+spec:
+ nodeSelector:
+ beta.kubernetes.io/os: linux
+ containers:
+ - image: nginx
+ name: nginx-azuredisk
+ command:
+ - "/bin/sh"
+ - "-c"
+ - while true; do echo $(date) >> /mnt/azuredisk/outfile; sleep 1; done
+ volumeMounts:
+ - name: azuredisk
+ mountPath: "/mnt/azuredisk"
+ volumes:
+ - name: azuredisk
+ persistentVolumeClaim:
+ claimName: pvc-azuredisk
+```
diff --git a/inventory/sample/group_vars/all/azure.yml b/inventory/sample/group_vars/all/azure.yml
index 8b0313fde..02ea0f91a 100644
--- a/inventory/sample/group_vars/all/azure.yml
+++ b/inventory/sample/group_vars/all/azure.yml
@@ -14,3 +14,23 @@
# azure_route_table_name:
# supported values are 'standard' or 'vmss'
# azure_vmtype: standard
+
+## Azure Disk CSI credentials and parameters
+## see docs/azure-csi.md for details on how to get these values
+
+# azure_csi_tenant_id:
+# azure_csi_subscription_id:
+# azure_csi_aad_client_id:
+# azure_csi_aad_client_secret:
+# azure_csi_location:
+# azure_csi_resource_group:
+# azure_csi_vnet_name:
+# azure_csi_vnet_resource_group:
+# azure_csi_subnet_name:
+# azure_csi_security_group_name:
+# azure_csi_use_instance_metadata:
+
+## To enable Azure Disk CSI, uncomment below
+# azure_csi_enabled: true
+# azure_csi_controller_replicas: 1
+# azure_csi_plugin_image_tag: latest
diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml
index d30857d50..233c050e8 100644
--- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml
@@ -255,7 +255,8 @@ podsecuritypolicy_enabled: false
## See https://github.com/kubernetes-sigs/kubespray/issues/2141
## Set this variable to true to get rid of this issue
volume_cross_zone_attachment: false
-# Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI, AWS EBS CSI, GCP Persistent Disk CSI)
+## Add Persistent Volumes Storage Class for corresponding cloud provider (supported: in-tree OpenStack, Cinder CSI,
+## AWS EBS CSI, Azure Disk CSI, GCP Persistent Disk CSI)
persistent_volumes_enabled: false
## Container Engine Acceleration
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index b402ca079..7bbb92d76 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -531,6 +531,17 @@ cinder_csi_plugin_image_tag: "latest"
aws_ebs_csi_plugin_image_repo: "{{ docker_image_repo }}/amazon/aws-ebs-csi-driver"
aws_ebs_csi_plugin_image_tag: "latest"
+azure_csi_image_repo: "mcr.microsoft.com/oss/kubernetes-csi"
+azure_csi_provisioner_image_tag: "v1.5.0"
+azure_csi_attacher_image_tag: "v1.2.0"
+azure_csi_cluster_registrar_image_tag: "v1.0.1"
+azure_csi_node_registrar_image_tag: "v1.1.0"
+azure_csi_snapshotter_image_tag: "v2.0.0"
+azure_csi_resizer_image_tag: "v0.3.0"
+azure_csi_livenessprobe_image_tag: "v1.1.0"
+azure_csi_plugin_image_repo: "mcr.microsoft.com/k8s/csi"
+azure_csi_plugin_image_tag: "latest"
+
gcp_pd_csi_image_repo: "gke.gcr.io"
gcp_pd_csi_driver_image_tag: "v0.7.0-gke.0"
gcp_pd_csi_provisioner_image_tag: "v1.5.0-gke.0"
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml
new file mode 100644
index 000000000..c1eec6401
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/defaults/main.yml
@@ -0,0 +1,4 @@
+---
+azure_csi_use_instance_metadata: true
+azure_csi_controller_replicas: 1
+azure_csi_plugin_image_tag: latest
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml
new file mode 100644
index 000000000..0a858ee75
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/azure-credential-check.yml
@@ -0,0 +1,54 @@
+---
+- name: Azure CSI Driver | check azure_csi_tenant_id value
+ fail:
+ msg: "azure_csi_tenant_id is missing"
+ when: azure_csi_tenant_id is not defined or not azure_csi_tenant_id
+
+- name: Azure CSI Driver | check azure_csi_subscription_id value
+ fail:
+ msg: "azure_csi_subscription_id is missing"
+ when: azure_csi_subscription_id is not defined or not azure_csi_subscription_id
+
+- name: Azure CSI Driver | check azure_csi_aad_client_id value
+ fail:
+ msg: "azure_csi_aad_client_id is missing"
+ when: azure_csi_aad_client_id is not defined or not azure_csi_aad_client_id
+
+- name: Azure CSI Driver | check azure_csi_aad_client_secret value
+ fail:
+ msg: "azure_csi_aad_client_secret is missing"
+ when: azure_csi_aad_client_secret is not defined or not azure_csi_aad_client_secret
+
+- name: Azure CSI Driver | check azure_csi_resource_group value
+ fail:
+ msg: "azure_csi_resource_group is missing"
+ when: azure_csi_resource_group is not defined or not azure_csi_resource_group
+
+- name: Azure CSI Driver | check azure_csi_location value
+ fail:
+ msg: "azure_csi_location is missing"
+ when: azure_csi_location is not defined or not azure_csi_location
+
+- name: Azure CSI Driver | check azure_csi_subnet_name value
+ fail:
+ msg: "azure_csi_subnet_name is missing"
+ when: azure_csi_subnet_name is not defined or not azure_csi_subnet_name
+
+- name: Azure CSI Driver | check azure_csi_security_group_name value
+ fail:
+ msg: "azure_csi_security_group_name is missing"
+ when: azure_csi_security_group_name is not defined or not azure_csi_security_group_name
+
+- name: Azure CSI Driver | check azure_csi_vnet_name value
+ fail:
+ msg: "azure_csi_vnet_name is missing"
+ when: azure_csi_vnet_name is not defined or not azure_csi_vnet_name
+
+- name: Azure CSI Driver | check azure_csi_vnet_resource_group value
+ fail:
+ msg: "azure_csi_vnet_resource_group is missing"
+ when: azure_csi_vnet_resource_group is not defined or not azure_csi_vnet_resource_group
+
+- name: "Azure CSI Driver | check azure_csi_use_instance_metadata is a bool"
+ assert:
+ that: azure_csi_use_instance_metadata | type_debug == 'bool'
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml
new file mode 100644
index 000000000..e33ca292f
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/tasks/main.yml
@@ -0,0 +1,48 @@
+---
+- include_tasks: azure-credential-check.yml
+ tags: azure-csi-driver
+
+- name: Azure CSI Driver | Write Azure CSI cloud-config
+ template:
+ src: "azure-csi-cloud-config.j2"
+ dest: "{{ kube_config_dir }}/azure_csi_cloud_config"
+ group: "{{ kube_cert_group }}"
+ mode: 0640
+ when: inventory_hostname == groups['kube-master'][0]
+ tags: azure-csi-driver
+
+- name: Azure CSI Driver | Get base64 cloud-config
+ slurp:
+ src: "{{ kube_config_dir }}/azure_csi_cloud_config"
+ register: cloud_config_secret
+ when: inventory_hostname == groups['kube-master'][0]
+ tags: azure-csi-driver
+
+- name: Azure CSI Driver | Generate Manifests
+ template:
+ src: "{{ item.file }}.j2"
+ dest: "{{ kube_config_dir }}/{{ item.file }}"
+ with_items:
+ - {name: azure-csi-azuredisk-driver, file: azure-csi-azuredisk-driver.yml}
+ - {name: azure-csi-cloud-config-secret, file: azure-csi-cloud-config-secret.yml}
+ - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller-rbac.yml}
+ - {name: azure-csi-azuredisk-controller, file: azure-csi-azuredisk-controller.yml}
+ - {name: azure-csi-azuredisk-node, file: azure-csi-azuredisk-node.yml}
+ - {name: azure-csi-node-info-crd.yml.j2, file: azure-csi-node-info-crd.yml}
+ register: azure_csi_manifests
+ when: inventory_hostname == groups['kube-master'][0]
+ tags: azure-csi-driver
+
+- name: Azure CSI Driver | Apply Manifests
+ kube:
+ kubectl: "{{ bin_dir }}/kubectl"
+ filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+ state: "latest"
+ with_items:
+ - "{{ azure_csi_manifests.results }}"
+ when:
+ - inventory_hostname == groups['kube-master'][0]
+ - not item is skipped
+ loop_control:
+ label: "{{ item.item.file }}"
+ tags: azure-csi-driver
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2
new file mode 100644
index 000000000..ad974d38c
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller-rbac.yml.j2
@@ -0,0 +1,212 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: csi-azuredisk-controller-sa
+ namespace: kube-system
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-external-provisioner-role
+rules:
+ - apiGroups: [""]
+ resources: ["persistentvolumes"]
+ verbs: ["get", "list", "watch", "create", "delete"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["storageclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["csinodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshots"]
+ verbs: ["get", "list"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents"]
+ verbs: ["get", "list"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-csi-provisioner-binding
+subjects:
+ - kind: ServiceAccount
+ name: csi-azuredisk-controller-sa
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ name: azuredisk-external-provisioner-role
+ apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-external-attacher-role
+rules:
+ - apiGroups: [""]
+ resources: ["persistentvolumes"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: [""]
+ resources: ["nodes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["csi.storage.k8s.io"]
+ resources: ["csinodeinfos"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["volumeattachments"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-csi-attacher-binding
+subjects:
+ - kind: ServiceAccount
+ name: csi-azuredisk-controller-sa
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ name: azuredisk-external-attacher-role
+ apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-cluster-driver-registrar-role
+rules:
+ - apiGroups: ["apiextensions.k8s.io"]
+ resources: ["customresourcedefinitions"]
+ verbs: ["create", "list", "watch", "delete"]
+ - apiGroups: ["csi.storage.k8s.io"]
+ resources: ["csidrivers"]
+ verbs: ["create", "delete"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-csi-driver-registrar-binding
+subjects:
+ - kind: ServiceAccount
+ name: csi-azuredisk-controller-sa
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ name: azuredisk-cluster-driver-registrar-role
+ apiGroup: rbac.authorization.k8s.io
+
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-external-snapshotter-role
+rules:
+ - apiGroups: [""]
+ resources: ["persistentvolumes"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["storage.k8s.io"]
+ resources: ["storageclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["list", "watch", "create", "update", "patch"]
+ - apiGroups: [""]
+ resources: ["secrets"]
+ verbs: ["get", "list"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotclasses"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents"]
+ verbs: ["create", "get", "list", "watch", "update", "delete"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshots"]
+ verbs: ["get", "list", "watch", "update"]
+ - apiGroups: ["apiextensions.k8s.io"]
+ resources: ["customresourcedefinitions"]
+ verbs: ["create", "list", "watch", "delete"]
+ - apiGroups: ["snapshot.storage.k8s.io"]
+ resources: ["volumesnapshotcontents/status"]
+ verbs: ["update"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "watch", "list", "delete", "update", "create"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-csi-snapshotter-binding
+subjects:
+ - kind: ServiceAccount
+ name: csi-azuredisk-controller-sa
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ name: azuredisk-external-snapshotter-role
+ apiGroup: rbac.authorization.k8s.io
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-external-resizer-role
+rules:
+ - apiGroups: [""]
+ resources: ["persistentvolumes"]
+ verbs: ["get", "list", "watch", "update", "patch"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims"]
+ verbs: ["get", "list", "watch"]
+ - apiGroups: [""]
+ resources: ["persistentvolumeclaims/status"]
+ verbs: ["update", "patch"]
+ - apiGroups: [""]
+ resources: ["events"]
+ verbs: ["list", "watch", "create", "update", "patch"]
+ - apiGroups: ["coordination.k8s.io"]
+ resources: ["leases"]
+ verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: azuredisk-csi-resizer-role
+subjects:
+ - kind: ServiceAccount
+ name: csi-azuredisk-controller-sa
+ namespace: kube-system
+roleRef:
+ kind: ClusterRole
+ name: azuredisk-external-resizer-role
+ apiGroup: rbac.authorization.k8s.io
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
new file mode 100644
index 000000000..b9cd4dc4f
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-controller.yml.j2
@@ -0,0 +1,200 @@
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+ name: csi-azuredisk-controller
+ namespace: kube-system
+spec:
+ replicas: {{ azure_csi_controller_replicas }}
+ selector:
+ matchLabels:
+ app: csi-azuredisk-controller
+ template:
+ metadata:
+ labels:
+ app: csi-azuredisk-controller
+ spec:
+ hostNetwork: true
+ serviceAccountName: csi-azuredisk-controller-sa
+ nodeSelector:
+ beta.kubernetes.io/os: linux
+ priorityClassName: system-cluster-critical
+ tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Equal"
+ value: "true"
+ effect: "NoSchedule"
+ containers:
+ - name: csi-provisioner
+ image: {{ azure_csi_image_repo }}/csi-provisioner:{{ azure_csi_provisioner_image_tag }}
+ args:
+ - "--provisioner=disk.csi.azure.com"
+ - "--feature-gates=Topology=true"
+ - "--csi-address=$(ADDRESS)"
+ - "--connection-timeout=15s"
+ - "--v=5"
+ - "--timeout=120s"
+ - "--enable-leader-election"
+ - "--leader-election-type=leases"
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /csi
+ name: socket-dir
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: csi-attacher
+ image: {{ azure_csi_image_repo }}/csi-attacher:{{ azure_csi_attacher_image_tag }}
+ args:
+ - "-v=5"
+ - "-csi-address=$(ADDRESS)"
+ - "-timeout=120s"
+ - "-leader-election"
+ - "-leader-election-type=leases"
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /csi
+ name: socket-dir
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: cluster-driver-registrar
+ image: {{ azure_csi_image_repo }}/csi-cluster-driver-registrar:{{ azure_csi_cluster_registrar_image_tag }}
+ args:
+ - --csi-address=$(ADDRESS)
+ - --driver-requires-attachment=true
+ - --v=5
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: csi-snapshotter
+ image: {{ azure_csi_image_repo }}/csi-snapshotter:{{ azure_csi_snapshotter_image_tag }}
+ args:
+ - "-csi-address=$(ADDRESS)"
+ - "-leader-election"
+ - "--v=5"
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: csi-resizer
+ image: {{ azure_csi_image_repo }}/csi-resizer:{{ azure_csi_resizer_image_tag }}
+ args:
+ - "-csi-address=$(ADDRESS)"
+ - "-v=5"
+ - "-leader-election"
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: liveness-probe
+ image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}
+ args:
+ - --csi-address=/csi/csi.sock
+ - --connection-timeout=3s
+ - --health-port=29602
+ - --v=5
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: azuredisk
+ image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }}
+ args:
+ - "--v=5"
+ - "--endpoint=$(CSI_ENDPOINT)"
+ - "--nodeid=$(KUBE_NODE_NAME)"
+ ports:
+ - containerPort: 29602
+ name: healthz
+ protocol: TCP
+ - containerPort: 29604
+ name: metrics
+ protocol: TCP
+ livenessProbe:
+ failureThreshold: 5
+ httpGet:
+ path: /healthz
+ port: healthz
+ initialDelaySeconds: 30
+ timeoutSeconds: 10
+ periodSeconds: 30
+ env:
+ - name: AZURE_CREDENTIAL_FILE
+ value: "/etc/kubernetes/azure.json"
+ - name: CSI_ENDPOINT
+ value: unix:///csi/csi.sock
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /csi
+ name: socket-dir
+ - mountPath: /etc/kubernetes/
+ name: azure-cred
+ readOnly: true
+ - mountPath: /var/lib/waagent/ManagedIdentity-Settings
+ readOnly: true
+ name: msi
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ volumes:
+ - name: socket-dir
+ emptyDir: {}
+ - name: azure-cred
+ secret:
+ secretName: cloud-config
+ - name: msi
+ hostPath:
+ path: /var/lib/waagent/ManagedIdentity-Settings
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2
new file mode 100644
index 000000000..4c24cc244
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-driver.yml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: storage.k8s.io/v1beta1
+kind: CSIDriver
+metadata:
+ name: disk.csi.azure.com
+spec:
+ attachRequired: true
+ podInfoOnMount: true
+ volumeLifecycleModes: # added in Kubernetes 1.16
+ - Persistent
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2
new file mode 100644
index 000000000..08957781a
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-azuredisk-node.yml.j2
@@ -0,0 +1,156 @@
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+ name: csi-azuredisk-node
+ namespace: kube-system
+spec:
+ selector:
+ matchLabels:
+ app: csi-azuredisk-node
+ template:
+ metadata:
+ labels:
+ app: csi-azuredisk-node
+ spec:
+ hostNetwork: true
+ nodeSelector:
+ beta.kubernetes.io/os: linux
+ priorityClassName: system-node-critical
+ containers:
+ - name: liveness-probe
+ imagePullPolicy: IfNotPresent
+ volumeMounts:
+ - mountPath: /csi
+ name: socket-dir
+ image: {{ azure_csi_image_repo }}/livenessprobe:{{ azure_csi_livenessprobe_image_tag }}
+ args:
+ - --csi-address=/csi/csi.sock
+ - --connection-timeout=3s
+ - --health-port=29603
+ - --v=5
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: node-driver-registrar
+ image: {{ azure_csi_image_repo }}/csi-node-driver-registrar:{{ azure_csi_node_registrar_image_tag }}
+ args:
+ - --csi-address=$(ADDRESS)
+ - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+ - --v=5
+ lifecycle:
+ preStop:
+ exec:
+ command: ["/bin/sh", "-c", "rm -rf /registration/disk.csi.azure.com-reg.sock /csi/csi.sock"]
+ env:
+ - name: ADDRESS
+ value: /csi/csi.sock
+ - name: DRIVER_REG_SOCK_PATH
+ value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi.sock
+ volumeMounts:
+ - name: socket-dir
+ mountPath: /csi
+ - name: registration-dir
+ mountPath: /registration
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ - name: azuredisk
+ image: {{ azure_csi_plugin_image_repo }}/azuredisk-csi:{{ azure_csi_plugin_image_tag }}
+ args:
+ - "--v=5"
+ - "--endpoint=$(CSI_ENDPOINT)"
+ - "--nodeid=$(KUBE_NODE_NAME)"
+ - "--metrics-address=0.0.0.0:29605"
+ ports:
+ - containerPort: 29603
+ name: healthz
+ protocol: TCP
+ - containerPort: 29605
+ name: metrics
+ protocol: TCP
+ livenessProbe:
+ failureThreshold: 5
+ httpGet:
+ path: /healthz
+ port: healthz
+ initialDelaySeconds: 30
+ timeoutSeconds: 10
+ periodSeconds: 30
+ env:
+ - name: AZURE_CREDENTIAL_FILE
+ value: "/etc/kubernetes/azure.json"
+ - name: CSI_ENDPOINT
+ value: unix:///csi/csi.sock
+ - name: KUBE_NODE_NAME
+ valueFrom:
+ fieldRef:
+ apiVersion: v1
+ fieldPath: spec.nodeName
+ imagePullPolicy: IfNotPresent
+ securityContext:
+ privileged: true
+ volumeMounts:
+ - mountPath: /csi
+ name: socket-dir
+ - mountPath: /var/lib/kubelet/
+ mountPropagation: Bidirectional
+ name: mountpoint-dir
+ - mountPath: /etc/kubernetes/
+ name: azure-cred
+ readOnly: true
+ - mountPath: /var/lib/waagent/ManagedIdentity-Settings
+ readOnly: true
+ name: msi
+ - mountPath: /dev
+ name: device-dir
+ - mountPath: /sys/bus/scsi/devices
+ name: sys-devices-dir
+ - mountPath: /sys/class/scsi_host/
+ name: scsi-host-dir
+ resources:
+ limits:
+ cpu: 200m
+ memory: 200Mi
+ requests:
+ cpu: 10m
+ memory: 20Mi
+ volumes:
+ - hostPath:
+ path: /var/lib/kubelet/plugins/disk.csi.azure.com
+ type: DirectoryOrCreate
+ name: socket-dir
+ - hostPath:
+ path: /var/lib/kubelet/
+ type: DirectoryOrCreate
+ name: mountpoint-dir
+ - hostPath:
+ path: /var/lib/kubelet/plugins_registry/
+ type: DirectoryOrCreate
+ name: registration-dir
+ - name: azure-cred
+ secret:
+ secretName: cloud-config
+ - hostPath:
+ path: /var/lib/waagent/ManagedIdentity-Settings
+ name: msi
+ - hostPath:
+ path: /dev
+ type: Directory
+ name: device-dir
+ - hostPath:
+ path: /sys/bus/scsi/devices
+ type: Directory
+ name: sys-devices-dir
+ - hostPath:
+ path: /sys/class/scsi_host/
+ type: Directory
+ name: scsi-host-dir
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2
new file mode 100644
index 000000000..f259cec9f
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config-secret.yml.j2
@@ -0,0 +1,7 @@
+kind: Secret
+apiVersion: v1
+metadata:
+ name: cloud-config
+ namespace: kube-system
+data:
+ azure.json: {{ cloud_config_secret.content }}
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2
new file mode 100644
index 000000000..d3932f50d
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-cloud-config.j2
@@ -0,0 +1,14 @@
+{
+ "cloud":"AzurePublicCloud",
+ "tenantId": "{{ azure_csi_tenant_id }}",
+ "subscriptionId": "{{ azure_csi_subscription_id }}",
+ "aadClientId": "{{ azure_csi_aad_client_id }}",
+ "aadClientSecret": "{{ azure_csi_aad_client_secret }}",
+ "location": "{{ azure_csi_location }}",
+ "resourceGroup": "{{ azure_csi_resource_group }}",
+ "vnetName": "{{ azure_csi_vnet_name }}",
+ "vnetResourceGroup": "{{ azure_csi_vnet_resource_group }}",
+ "subnetName": "{{ azure_csi_subnet_name }}",
+ "securityGroupName": "{{ azure_csi_security_group_name }}",
+ "useInstanceMetadata": {{ azure_csi_use_instance_metadata }},
+}
diff --git a/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2 b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2
new file mode 100644
index 000000000..4ac7ff056
--- /dev/null
+++ b/roles/kubernetes-apps/csi_driver/azuredisk/templates/azure-csi-node-info-crd.yml.j2
@@ -0,0 +1,38 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ creationTimestamp: null
+ name: csinodeinfos.csi.storage.k8s.io
+spec:
+ group: csi.storage.k8s.io
+ names:
+ kind: CSINodeInfo
+ plural: csinodeinfos
+ scope: Cluster
+ validation:
+ openAPIV3Schema:
+ properties:
+ csiDrivers:
+ description: List of CSI drivers running on the node and their properties.
+ items:
+ properties:
+ driver:
+ description: The CSI driver that this object refers to.
+ type: string
+ nodeID:
+ description: The node from the driver point of view.
+ type: string
+ topologyKeys:
+ description: List of keys supported by the driver.
+ items:
+ type: string
+ type: array
+ type: array
+ version: v1alpha1
+status:
+ acceptedNames:
+ kind: ""
+ plural: ""
+ conditions: []
+ storedVersions: []
diff --git a/roles/kubernetes-apps/meta/main.yml b/roles/kubernetes-apps/meta/main.yml
index 8e5758f7e..34fd366a3 100644
--- a/roles/kubernetes-apps/meta/main.yml
+++ b/roles/kubernetes-apps/meta/main.yml
@@ -45,6 +45,14 @@ dependencies:
- aws-ebs-csi-driver
- csi-driver
+ - role: kubernetes-apps/csi_driver/azuredisk
+ when:
+ - azure_csi_enabled
+ tags:
+ - apps
+ - azure-csi-driver
+ - csi-driver
+
- role: kubernetes-apps/csi_driver/gcp_pd
when:
- gcp_pd_csi_enabled
diff --git a/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml
new file mode 100644
index 000000000..fc92e17b5
--- /dev/null
+++ b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+## Available values: Standard_LRS, Premium_LRS, StandardSSD_LRS, UltraSSD_LRS
+storage_account_type: StandardSSD_LRS
diff --git a/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml
new file mode 100644
index 000000000..8a35dbf61
--- /dev/null
+++ b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/tasks/main.yml
@@ -0,0 +1,19 @@
+---
+- name: Kubernetes Persistent Volumes | Copy Azure CSI Storage Class template
+ template:
+ src: "azure-csi-storage-class.yml.j2"
+ dest: "{{ kube_config_dir }}/azure-csi-storage-class.yml"
+ register: manifests
+ when:
+ - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Persistent Volumes | Add Azure CSI Storage Class
+ kube:
+ name: cinder-csi
+ kubectl: "{{ bin_dir }}/kubectl"
+ resource: StorageClass
+ filename: "{{ kube_config_dir }}/azure-csi-storage-class.yml"
+ state: "latest"
+ when:
+ - inventory_hostname == groups['kube-master'][0]
+ - manifests.changed
diff --git a/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2 b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2
new file mode 100644
index 000000000..80f02b3db
--- /dev/null
+++ b/roles/kubernetes-apps/persistent_volumes/azuredisk-csi/templates/azure-csi-storage-class.yml.j2
@@ -0,0 +1,11 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+ name: disk.csi.azure.com
+provisioner: disk.csi.azure.com
+parameters:
+ skuname: {{ storage_account_type }}
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+allowVolumeExpansion: true
diff --git a/roles/kubernetes-apps/persistent_volumes/meta/main.yml b/roles/kubernetes-apps/persistent_volumes/meta/main.yml
index 42096aa7b..c0522df9c 100644
--- a/roles/kubernetes-apps/persistent_volumes/meta/main.yml
+++ b/roles/kubernetes-apps/persistent_volumes/meta/main.yml
@@ -21,6 +21,13 @@ dependencies:
- persistent_volumes_aws_ebs_csi
- aws-ebs-csi-driver
+ - role: kubernetes-apps/persistent_volumes/azuredisk-csi
+ when:
+ - azure_csi_enabled
+ tags:
+ - persistent_volumes_azure_csi
+ - azure-csi-driver
+
- role: kubernetes-apps/persistent_volumes/gcp-pd-csi
when:
- gcp_pd_csi_enabled
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index e2ada4e9c..6c05d7152 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -305,6 +305,7 @@ local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }
local_volume_provisioner_directory_mode: 0700
cinder_csi_enabled: false
aws_ebs_csi_enabled: false
+azure_csi_enabled: false
gcp_pd_csi_enabled: false
persistent_volumes_enabled: false
cephfs_provisioner_enabled: false
--
GitLab