Skip to content
Snippets Groups Projects
Unverified Commit 646fd5f4 authored by Ali Sanhaji's avatar Ali Sanhaji Committed by GitHub
Browse files

External OpenStack Cloud Controller Manager implementation (#5491)

* External OpenStack Cloud Controller Manager implementation

* Adding controller image tag

* Minor fixes

* Restructuring the external cloud controller to work with KubeADM
parent 277b3476
No related branches found
No related tags found
No related merge requests found
Showing
with 473 additions and 16 deletions
......@@ -97,6 +97,7 @@
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles:
- { role: kubespray-defaults}
- { role: kubernetes-apps/external_cloud_controller, tags: external-cloud-controller }
- { role: kubernetes-apps/network_plugin, tags: network }
- { role: kubernetes-apps/policy_controller, tags: policy-controller }
- { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
......
......@@ -51,10 +51,13 @@ loadbalancer_apiserver_healthcheck_port: 8081
## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', 'oci', or 'external'
## When openstack is used make sure to source in the openstack credentials
## like you would do when using openstack-client before starting the playbook.
## Note: The 'external' cloud provider is not supported.
## TODO(riverzhang): https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#running-cloud-controller-manager
# cloud_provider:
## When cloud_provider is set to 'external', you can set the cloud controller to deploy
## Supported cloud controllers are: 'openstack'
## When openstack is used make sure to source in the openstack credentials
# external_cloud_provider:
## Set these proxy values in order to update package manager and docker daemon to use proxies
# http_proxy: ""
# https_proxy: ""
......
......@@ -15,6 +15,21 @@
# openstack_lbaas_monitor_timeout: "30s"
# openstack_lbaas_monitor_max_retries: "3"
## Values for the external OpenStack Cloud Controller
# external_openstack_lbaas_network_id: "Neutron network ID to create LBaaS VIP"
# external_openstack_lbaas_subnet_id: "Neutron subnet ID to create LBaaS VIP"
# external_openstack_lbaas_floating_network_id: "Neutron network ID to get floating IP from"
# external_openstack_lbaas_floating_subnet_id: "Neutron subnet ID to get floating IP from"
# external_openstack_lbaas_use_octavia: true
# external_openstack_lbaas_method: "ROUND_ROBIN"
# external_openstack_lbaas_create_monitor: false
# external_openstack_lbaas_monitor_delay: "1m"
# external_openstack_lbaas_monitor_timeout: "30s"
# external_openstack_lbaas_monitor_max_retries: "3"
## The tag of the external OpenStack Cloud Controller image
# external_openstack_cloud_controller_image_tag: "latest"
## To use Cinder CSI plugin to provision volumes set this value to true
## Make sure to source in the openstack credentials
# cinder_csi_enabled: true
......
---
dependencies:
- role: kubernetes-apps/external_cloud_controller/openstack
when:
- cloud_provider is defined
- cloud_provider == "external"
- external_cloud_provider is defined
- external_cloud_provider == "openstack"
- inventory_hostname == groups['kube-master'][0]
tags:
- external-cloud-controller
- external-openstack
---
# The external cloud controller will need credentials to access
# openstack apis. Per default these values will be
# read from the environment.
external_openstack_auth_url: "{{ lookup('env','OS_AUTH_URL') }}"
external_openstack_username: "{{ lookup('env','OS_USERNAME') }}"
external_openstack_password: "{{ lookup('env','OS_PASSWORD') }}"
external_openstack_region: "{{ lookup('env','OS_REGION_NAME') }}"
external_openstack_tenant_id: "{{ lookup('env','OS_TENANT_ID')| default(lookup('env','OS_PROJECT_ID'),true) }}"
external_openstack_tenant_name: "{{ lookup('env','OS_TENANT_NAME') }}"
external_openstack_domain_name: "{{ lookup('env','OS_USER_DOMAIN_NAME') }}"
external_openstack_domain_id: "{{ lookup('env','OS_USER_DOMAIN_ID') }}"
external_openstack_cacert: "{{ lookup('env','OS_CACERT') }}"
external_openstack_cloud_controller_image_tag: "latest"
---
- include_tasks: openstack-credential-check.yml
tags: external-openstack
- name: External OpenStack Cloud Controller | Write cacert file
copy:
src: "{{ external_openstack_cacert }}"
dest: "{{ kube_config_dir }}/external-openstack-cacert.pem"
group: "{{ kube_cert_group }}"
mode: 0640
when:
- inventory_hostname in groups['k8s-cluster']
- external_openstack_cacert is defined
- external_openstack_cacert | length > 0
tags: external-openstack
- name: External OpenStack Cloud Controller | Write External OpenStack cloud-config
template:
src: "external-openstack-cloud-config.j2"
dest: "{{ kube_config_dir }}/external_openstack_cloud_config"
group: "{{ kube_cert_group }}"
mode: 0640
when: inventory_hostname == groups['kube-master'][0]
tags: external-openstack
- name: External OpenStack Cloud Controller | Get base64 cloud-config
slurp:
src: "{{ kube_config_dir }}/external_openstack_cloud_config"
register: external_openstack_cloud_config_secret
when: inventory_hostname == groups['kube-master'][0]
tags: external-openstack
- name: External OpenStack Cloud Controller | Generate Manifests
template:
src: "{{ item.file }}.j2"
dest: "{{ kube_config_dir }}/{{ item.file }}"
with_items:
- {name: external-openstack-cloud-config-secret, file: external-openstack-cloud-config-secret.yml}
- {name: external-openstack-cloud-controller-manager-roles, file: external-openstack-cloud-controller-manager-roles.yml}
- {name: external-openstack-cloud-controller-manager-role-bindings, file: external-openstack-cloud-controller-manager-role-bindings.yml}
- {name: external-openstack-cloud-controller-manager-ds, file: external-openstack-cloud-controller-manager-ds.yml}
register: external_openstack_manifests
when: inventory_hostname == groups['kube-master'][0]
tags: external-openstack
- name: External OpenStack Cloud Controller | Apply Manifests
kube:
kubectl: "{{ bin_dir }}/kubectl"
filename: "{{ kube_config_dir }}/{{ item.item.file }}"
state: "latest"
with_items:
- "{{ external_openstack_manifests.results }}"
when:
- inventory_hostname == groups['kube-master'][0]
- not item is skipped
loop_control:
label: "{{ item.item.file }}"
tags: external-openstack
---
- name: External OpenStack Cloud Controller | check external_openstack_auth_url value
fail:
msg: "external_openstack_auth_url is missing"
when: external_openstack_auth_url is not defined or not external_openstack_auth_url
- name: External OpenStack Cloud Controller | check external_openstack_username value
fail:
msg: "external_openstack_username is missing"
when: external_openstack_username is not defined or not external_openstack_username
- name: External OpenStack Cloud Controller | check external_openstack_password value
fail:
msg: "external_openstack_password is missing"
when: external_openstack_password is not defined or not external_openstack_password
- name: External OpenStack Cloud Controller | check external_openstack_region value
fail:
msg: "external_openstack_region is missing"
when: external_openstack_region is not defined or not external_openstack_region
- name: External OpenStack Cloud Controller | check external_openstack_tenant_id value
fail:
msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
when:
- external_openstack_tenant_id is not defined or not external_openstack_tenant_id
- external_openstack_tenant_name is not defined
- name: External OpenStack Cloud Controller | check external_openstack_tenant_name value
fail:
msg: "one of external_openstack_tenant_id or external_openstack_tenant_name must be specified"
when:
- external_openstack_tenant_name is not defined or not external_openstack_tenant_name
- external_openstack_tenant_id is not defined
# This YAML file contains secret objects,
# which are necessary to run external openstack cloud controller.
kind: Secret
apiVersion: v1
metadata:
name: external-openstack-cloud-config
namespace: kube-system
data:
cloud.conf: {{ external_openstack_cloud_config_secret.content }}
[Global]
auth-url="{{ external_openstack_auth_url }}"
username="{{ external_openstack_username }}"
password="{{ external_openstack_password }}"
region="{{ external_openstack_region }}"
{% if external_openstack_tenant_id is defined and external_openstack_tenant_id != "" %}
tenant-id="{{ external_openstack_tenant_id }}"
{% endif %}
{% if external_openstack_tenant_name is defined and external_openstack_tenant_name != "" %}
tenant-name="{{ external_openstack_tenant_name }}"
{% endif %}
{% if external_openstack_domain_name is defined and external_openstack_domain_name != "" %}
domain-name="{{ external_openstack_domain_name }}"
{% elif external_openstack_domain_id is defined and external_openstack_domain_id != "" %}
domain-id ="{{ external_openstack_domain_id }}"
{% endif %}
{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
ca-file="{{ kube_config_dir }}/external-openstack-cacert.pem"
{% endif %}
[LoadBalancer]
use-octavia={{ external_openstack_lbaas_use_octavia }}
create-monitor={{ openstack_lbaas_create_monitor }}
monitor-delay={{ openstack_lbaas_monitor_delay }}
monitor-timeout={{ openstack_lbaas_monitor_timeout }}
monitor-max-retries={{ openstack_lbaas_monitor_max_retries }}
{% if external_openstack_lbaas_method is defined %}
lb-method={{ external_openstack_lbaas_method }}
{% endif %}
{% if external_openstack_lbaas_network_id is defined %}
network-id={{ external_openstack_lbaas_network_id }}
{% endif %}
{% if external_openstack_lbaas_subnet_id is defined %}
subnet-id={{ external_openstack_lbaas_subnet_id }}
{% endif %}
{% if external_openstack_lbaas_floating_network_id is defined %}
floating-network-id={{ external_openstack_lbaas_floating_network_id }}
{% endif %}
{% if external_openstack_lbaas_flaoting_subnet_id is defined %}
floating-subnet-id={{ external_openstack_lbaas_floating_subnet_id }}
{% endif %}
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: openstack-cloud-controller-manager
namespace: kube-system
labels:
k8s-app: openstack-cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: openstack-cloud-controller-manager
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
k8s-app: openstack-cloud-controller-manager
spec:
nodeSelector:
node-role.kubernetes.io/master: ""
securityContext:
runAsUser: 1001
tolerations:
- key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
effect: NoSchedule
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: cloud-controller-manager
containers:
- name: openstack-cloud-controller-manager
image: {{ docker_image_repo }}/k8scloudprovider/openstack-cloud-controller-manager:{{ external_openstack_cloud_controller_image_tag }}
args:
- /bin/openstack-cloud-controller-manager
- --v=1
- --cloud-config=$(CLOUD_CONFIG)
- --cloud-provider=openstack
- --use-service-account-credentials=true
- --address=127.0.0.1
volumeMounts:
- mountPath: /etc/kubernetes/pki
name: k8s-certs
readOnly: true
- mountPath: /etc/ssl/certs
name: ca-certs
readOnly: true
- mountPath: /etc/config
name: cloud-config-volume
readOnly: true
{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
- mountPath: {{ kube_config_dir }}/external-openstack-cacert.pem
name: openstack-cacert
readOnly: true
{% endif %}
- mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
name: flexvolume-dir
resources:
requests:
cpu: 200m
env:
- name: CLOUD_CONFIG
value: /etc/config/cloud.conf
hostNetwork: true
volumes:
- hostPath:
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
type: DirectoryOrCreate
name: flexvolume-dir
- hostPath:
path: /etc/kubernetes/pki
type: DirectoryOrCreate
name: k8s-certs
- hostPath:
path: /etc/ssl/certs
type: DirectoryOrCreate
name: ca-certs
- name: cloud-config-volume
secret:
secretName: external-openstack-cloud-config
{% if external_openstack_cacert is defined and external_openstack_cacert != "" %}
- hostPath:
path: {{ kube_config_dir }}/external-openstack-cacert.pem
type: FileOrCreate
name: openstack-cacert
{% endif %}
apiVersion: v1
items:
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-node-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-node-controller
subjects:
- kind: ServiceAccount
name: cloud-node-controller
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:pvl-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:pvl-controller
subjects:
- kind: ServiceAccount
name: pvl-controller
namespace: kube-system
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:cloud-controller-manager
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
kind: List
metadata: {}
apiVersion: v1
items:
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-controller-manager
rules:
- apiGroups:
- coordination.k8s.io
resources:
- leases
verbs:
- get
- create
- update
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- services
verbs:
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
- serviceaccounts
verbs:
- create
- get
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- endpoints
verbs:
- create
- get
- list
- watch
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- secrets
verbs:
- list
- get
- watch
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:cloud-node-controller
rules:
- apiGroups:
- ""
resources:
- nodes
verbs:
- '*'
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:pvl-controller
rules:
- apiGroups:
- ""
resources:
- persistentvolumes
verbs:
- '*'
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- update
kind: List
metadata: {}
......@@ -21,6 +21,10 @@ nodeRegistration:
taints: []
{% endif %}
criSocket: {{ cri_socket }}
{% if cloud_provider is defined and cloud_provider in ["external"] %}
kubeletExtraArgs:
cloud-provider: external
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
......@@ -170,12 +174,10 @@ apiServer:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{cloud_provider}}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
......@@ -244,20 +246,18 @@ controllerManager:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{cloud_provider}}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
configure-cloud-routes: "false"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] or controller_manager_extra_volumes %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
- name: openstackcacert
hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
......
......@@ -24,6 +24,10 @@ nodeRegistration:
taints: []
{% endif %}
criSocket: {{ cri_socket }}
{% if cloud_provider is defined and cloud_provider in ["external"] %}
kubeletExtraArgs:
cloud-provider: external
{% endif %}
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
......@@ -173,12 +177,10 @@ apiServer:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{ cloud_provider }}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or kube_webhook_token_auth|default(false) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes or ssl_ca_dirs|length %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
......@@ -247,20 +249,18 @@ controllerManager:
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
cloud-provider: {{ cloud_provider }}
cloud-config: {{ kube_config_dir }}/cloud_config
{% elif cloud_provider is defined and cloud_provider in ["external"] %}
cloud-config: {{ kube_config_dir }}/cloud_config
{% endif %}
{% if kube_network_plugin is defined and kube_network_plugin not in ["cloud"] %}
configure-cloud-routes: "false"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] or controller_manager_extra_volumes %}
extraVolumes:
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
- name: openstackcacert
hostPath: "{{ kube_config_dir }}/openstack-cacert.pem"
mountPath: "{{ kube_config_dir }}/openstack-cacert.pem"
{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %}
{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %}
- name: cloud-config
hostPath: {{ kube_config_dir }}/cloud_config
mountPath: {{ kube_config_dir }}/cloud_config
......
......@@ -323,6 +323,13 @@ openstack_lbaas_monitor_timeout: "30s"
openstack_lbaas_monitor_max_retries: "3"
openstack_cacert: "{{ lookup('env','OS_CACERT') }}"
# Default values for the external OpenStack Cloud Controller
external_openstack_lbaas_use_octavia: true
external_openstack_lbaas_create_monitor: false
external_openstack_lbaas_monitor_delay: "1m"
external_openstack_lbaas_monitor_timeout: "30s"
external_openstack_lbaas_monitor_max_retries: "3"
## List of authorization modes that must be configured for
## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and
## 'RBAC' modes are tested. Order is important.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment