Add huawei cloud controller (#10198)

* Add huaweicloud as external cloud controller

* Add huaweicloud example config

* Rename AK,SK to ACCESS_KEY and SECRET_KEY

* Add reference to huaweicloud

* Fix variable naming

* Fix env var name

* Update example

* Fix variable naming

* Fix cloud_config path

* Add namespace for leader election

* Revert reviewers

* Delete OWNERS

Delete owners who are not responsible here.

* Fix build validation

inventory/sample/group_vars/all/all.yml

@@ -48,7 +48,7 @@ loadbalancer_apiserver_healthcheck_port: 8081
 # cloud_provider:
 
 ## When cloud_provider is set to 'external', you can set the cloud controller to deploy
-## Supported cloud controllers are: 'openstack', 'vsphere' and 'hcloud'
+## Supported cloud controllers are: 'openstack', 'vsphere', 'huaweicloud' and 'hcloud'
 ## When openstack or vsphere are used make sure to source in the required fields
 # external_cloud_provider:
 

inventory/sample/group_vars/all/huaweicloud.yml

+## Values for the external Huawei Cloud Controller
+# external_huaweicloud_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
+# external_huaweicloud_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
+
+## Credentials to authenticate against Keystone API
+## All of them are required Per default these values will be
+## read from the environment.
+# external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+# external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY')  }}"
+# external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY')  }}"
+# external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+# external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
+# external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}"
+
+## The repo and tag of the external Huawei Cloud Controller image
+# external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com"
+# external_huawei_cloud_controller_image_tag: "v0.26.3"

roles/kubernetes-apps/external_cloud_controller/huaweicloud/defaults/main.yml

+---
+# The external cloud controller will need credentials to access
+# openstack apis. Per default these values will be
+# read from the environment.
+external_huaweicloud_auth_url: "{{ lookup('env','OS_AUTH_URL')  }}"
+external_huaweicloud_access_key: "{{ lookup('env','OS_ACCESS_KEY')  }}"
+external_huaweicloud_secret_key: "{{ lookup('env','OS_SECRET_KEY')  }}"
+external_huaweicloud_region: "{{ lookup('env','OS_REGION_NAME')  }}"
+external_huaweicloud_project_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
+external_huaweicloud_cloud: "{{ lookup('env','OS_CLOUD') }}"
+
+## A dictionary of extra arguments to add to the huawei cloud controller manager deployment
+## Format:
+##  external_huawei_cloud_controller_extra_args:
+##    arg1: "value1"
+##    arg2: "value2"
+external_huawei_cloud_controller_extra_args: {}
+external_huawei_cloud_controller_image_repo: "swr.ap-southeast-1.myhuaweicloud.com"
+external_huawei_cloud_controller_image_tag: "v0.26.3"

roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/huaweicloud-credential-check.yml

+---
+- name: External Huawei Cloud Controller | check external_huaweicloud_auth_url value
+  fail:
+    msg: "external_huaweicloud_auth_url is missing"
+  when: external_huaweicloud_auth_url is not defined or not external_huaweicloud_auth_url
+
+
+- name: External Huawei Cloud Controller | check external_huaweicloud_access_key value
+  fail:
+    msg: "you must set external_huaweicloud_access_key"
+  when:
+    - external_huaweicloud_access_key is not defined or not external_huaweicloud_access_key
+
+- name: External Huawei Cloud Controller | check external_huaweicloud_secret_key value
+  fail:
+    msg: "external_huaweicloud_secret_key is missing"
+  when:
+    - external_huaweicloud_access_key is defined
+    - external_huaweicloud_access_key|length > 0
+    - external_huaweicloud_secret_key is not defined or not external_huaweicloud_secret_key
+
+
+- name: External Huawei Cloud Controller | check external_huaweicloud_region value
+  fail:
+    msg: "external_huaweicloud_region is missing"
+  when: external_huaweicloud_region is not defined or not external_huaweicloud_region
+
+
+- name: External Huawei Cloud Controller | check external_huaweicloud_project_id value
+  fail:
+    msg: "one of external_huaweicloud_project_id must be specified"
+  when:
+    - external_huaweicloud_project_id is not defined or not external_huaweicloud_project_id

roles/kubernetes-apps/external_cloud_controller/huaweicloud/tasks/main.yml

+---
+- name: External Huawei Cloud Controller | Check Huawei credentials
+  include_tasks: huaweicloud-credential-check.yml
+  tags: external-huaweicloud
+
+- name: External huaweicloud Cloud Controller | Get base64 cacert
+  slurp:
+    src: "{{ external_huaweicloud_cacert }}"
+  register: external_huaweicloud_cacert_b64
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - external_huaweicloud_cacert is defined
+    - external_huaweicloud_cacert | length > 0
+  tags: external-huaweicloud
+
+- name: External huaweicloud Cloud Controller | Get base64 cloud-config
+  set_fact:
+    external_huawei_cloud_config_secret: "{{ lookup('template', 'external-huawei-cloud-config.j2') | b64encode }}"
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  tags: external-huaweicloud
+
+- name: External Huawei Cloud Controller | Generate Manifests
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+    group: "{{ kube_cert_group }}"
+    mode: 0640
+  with_items:
+    - {name: external-huawei-cloud-config-secret, file: external-huawei-cloud-config-secret.yml}
+    - {name: external-huawei-cloud-controller-manager-roles, file: external-huawei-cloud-controller-manager-roles.yml}
+    - {name: external-huawei-cloud-controller-manager-role-bindings, file: external-huawei-cloud-controller-manager-role-bindings.yml}
+    - {name: external-huawei-cloud-controller-manager-ds, file: external-huawei-cloud-controller-manager-ds.yml}
+  register: external_huaweicloud_manifests
+  when: inventory_hostname == groups['kube_control_plane'][0]
+  tags: external-huaweicloud
+
+- name: External Huawei Cloud Controller | Apply Manifests
+  kube:
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/{{ item.item.file }}"
+    state: "latest"
+  with_items:
+    - "{{ external_huaweicloud_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube_control_plane'][0]
+    - not item is skipped
+  loop_control:
+    label: "{{ item.item.file }}"
+  tags: external-huaweicloud

roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config-secret.yml.j2

+# This YAML file contains secret objects,
+# which are necessary to run external huaweicloud cloud controller.
+
+kind: Secret
+apiVersion: v1
+metadata:
+  name: external-huawei-cloud-config
+  namespace: kube-system
+data:
+  cloud-config: {{ external_huawei_cloud_config_secret }}

roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-config.j2

+[Global]
+auth-url="{{ external_huaweicloud_auth_url }}"
+{% if external_huaweicloud_access_key is defined and external_huaweicloud_access_key != "" %}
+access-key={{ external_huaweicloud_access_key }}
+{% endif %}
+{% if external_huaweicloud_secret_key is defined and external_huaweicloud_secret_key != "" %}
+secret-key={{ external_huaweicloud_secret_key }}
+{% endif %}
+region="{{ external_huaweicloud_region }}"
+{% if external_huaweicloud_project_id is defined and external_huaweicloud_project_id != "" %}
+project-id="{{ external_huaweicloud_project_id }}"
+{% endif %}
+{% if external_huaweicloud_cloud is defined and external_huaweicloud_cloud != "" %}
+cloud="{{ external_huaweicloud_cloud }}"
+{% endif %}
+
+[VPC]
+{% if external_huaweicloud_lbaas_subnet_id is defined %}
+subnet-id={{ external_huaweicloud_lbaas_subnet_id }}
+{% endif %}
+{% if external_huaweicloud_lbaas_network_id is defined %}
+id={{ external_huaweicloud_lbaas_network_id }}
+{% endif %}

roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-ds.yml.j2

+kind: Namespace
+apiVersion: v1
+metadata:
+  name: huawei-cloud-provider
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: huawei-cloud-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: huawei-cloud-controller-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: huawei-cloud-controller-manager
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        k8s-app: huawei-cloud-controller-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      securityContext:
+        runAsUser: 1001
+      tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        effect: NoSchedule
+      serviceAccountName: cloud-controller-manager
+      containers:
+        - name: huawei-cloud-controller-manager
+          image: {{ external_huawei_cloud_controller_image_repo }}/k8s-cloudprovider/huawei-cloud-controller-manager:{{ external_huawei_cloud_controller_image_tag }}
+          args:
+            - /bin/huawei-cloud-controller-manager
+            - --v=1
+            - --cloud-config=$(CLOUD_CONFIG)
+            - --cloud-provider=huaweicloud
+            - --use-service-account-credentials=true
+{% for key, value in external_huawei_cloud_controller_extra_args.items() %}
+            - "{{ '--' + key + '=' + value }}"
+{% endfor %}
+          volumeMounts:
+            - mountPath: /etc/kubernetes
+              name: k8s-certs
+              readOnly: true
+            - mountPath: /etc/ssl/certs
+              name: ca-certs
+              readOnly: true
+            - mountPath: /etc/config
+              name: cloud-config-volume
+              readOnly: true
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+            - mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+              name: flexvolume-dir
+{% endif %}
+          resources:
+            requests:
+              cpu: 200m
+          env:
+            - name: CLOUD_CONFIG
+              value: /etc/config/cloud-config
+      hostNetwork: true
+      volumes:
+{% if kubelet_flexvolumes_plugins_dir is defined %}
+      - name: flexvolume-dir
+        hostPath:
+          path: "{{ kubelet_flexvolumes_plugins_dir }}"
+          type: DirectoryOrCreate
+{% endif %}
+      - name: k8s-certs
+        hostPath:
+          path: /etc/kubernetes
+          type: DirectoryOrCreate
+      - name: ca-certs
+        hostPath:
+          path: /etc/ssl/certs
+          type: DirectoryOrCreate
+      - name: cloud-config-volume
+        secret:
+          secretName: external-huawei-cloud-config

roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-role-bindings.yml.j2

+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRoleBinding
+  metadata:
+    name: system:cloud-controller-manager
+  roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: system:cloud-controller-manager
+  subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+kind: List
+metadata: {}

roles/kubernetes-apps/external_cloud_controller/huaweicloud/templates/external-huawei-cloud-controller-manager-roles.yml.j2

+apiVersion: v1
+items:
+- apiVersion: rbac.authorization.k8s.io/v1
+  kind: ClusterRole
+  metadata:
+    name: system:cloud-controller-manager
+  rules:
+  - resources:
+    - tokenreviews
+    verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    apiGroups:
+    - authentication.k8s.io
+  - resources:
+    - configmaps
+    - endpoints
+    - pods
+    - services
+    - secrets
+    - serviceaccounts
+    - serviceaccounts/token
+    verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    apiGroups:
+    - ''
+  - resources:
+    - nodes
+    verbs:
+    - get
+    - list
+    - watch
+    - delete
+    - patch
+    - update
+    apiGroups:
+    - ''
+  - resources:
+    - services/status
+    - pods/status
+    verbs:
+    - update
+    - patch
+    apiGroups:
+    - ''
+  - resources:
+    - nodes/status
+    verbs:
+    - patch
+    - update
+    apiGroups:
+    - ''
+  - resources:
+    - events
+    - endpoints
+    verbs:
+    - create
+    - patch
+    - update
+    apiGroups:
+    - ''
+  - resources:
+    - leases
+    verbs:
+    - get
+    - update
+    - create
+    - delete
+    apiGroups:
+    - coordination.k8s.io
+  - resources:
+    - customresourcedefinitions
+    verbs:
+    - get
+    - update
+    - create
+    - delete
+    apiGroups:
+      - apiextensions.k8s.io
+  - resources:
+    - ingresses
+    verbs:
+    - get
+    - list
+    - watch
+    - update
+    - create
+    - patch
+    - delete
+    apiGroups:
+    - networking.k8s.io
+  - resources:
+    - ingresses/status
+    verbs:
+    - update
+    - patch
+    apiGroups:
+    - networking.k8s.io
+  - resources:
+    - endpointslices
+    verbs:
+    - get
+    - list
+    - watch
+    apiGroups:
+    - discovery.k8s.io
+kind: List
+metadata: {}

roles/kubernetes-apps/external_cloud_controller/meta/main.yml

@@ -30,3 +30,13 @@ dependencies:
     tags:
       - external-cloud-controller
       - external-hcloud
+  - role: kubernetes-apps/external_cloud_controller/huaweicloud
+    when:
+      - cloud_provider is defined
+      - cloud_provider == "external"
+      - external_cloud_provider is defined
+      - external_cloud_provider == "huaweicloud"
+      - inventory_hostname == groups['kube_control_plane'][0]
+    tags:
+      - external-cloud-controller
+      - external-huaweicloud