diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml
index 9d2654c762a5f83f08c192a7a6cf4615f55bc6c3..1c938de199fd3868ee0de8032dc4e393ea9d7fe4 100644
--- a/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml
+++ b/inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml
@@ -24,3 +24,12 @@
# Advertise Cluster IPs
# calico_advertise_cluster_ips: true
+
+# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore)
+# calico_datastore: "etcd"
+
+# Use typha (only with kdd)
+# typha_enabled: false
+
+# Number of typha replicas
+# typha_replicas: 1
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index c483a5e3f497c01ec9b115f51ff6132ebc556106..a5080d97e53565859b4afbd4c19919b2c6b9b9a5 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -49,6 +49,7 @@ calico_ctl_version: "v3.4.4"
calico_cni_version: "v3.4.0"
calico_policy_version: "v3.4.0"
calico_rr_version: "v0.6.1"
+calico_typha_version: "v3.4.4"
flannel_version: "v0.11.0"
flannel_cni_version: "v0.3.0"
@@ -181,6 +182,8 @@ calico_policy_image_repo: "docker.io/calico/kube-controllers"
calico_policy_image_tag: "{{ calico_policy_version }}"
calico_rr_image_repo: "docker.io/calico/routereflector"
calico_rr_image_tag: "{{ calico_rr_version }}"
+calico_typha_image_repo: "docker.io/calico/typha"
+calico_typha_image_tag: "{{ calico_typha_version }}"
pod_infra_image_repo: "gcr.io/google_containers/pause-{{ image_arch }}"
pod_infra_image_tag: "{{ pod_infra_version }}"
install_socat_image_repo: "docker.io/xueshanf/install-socat"
@@ -434,6 +437,15 @@ downloads:
groups:
- calico-rr
+ calico_typha:
+ enabled: "{{ typha_enabled == 'calico' }}"
+ container: true
+ repo: "{{ calico_typha_image_repo }}"
+ tag: "{{ calico_typha_image_tag }}"
+ sha256: "{{ calico_typha_digest_checksum|default(None) }}"
+ groups:
+ - k8s-cluster
+
weave_kube:
enabled: "{{ kube_network_plugin == 'weave' }}"
container: true
diff --git a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
index 1be16776f8917cf8512ebcc4c7487fc71c3bf500..d82da98ae72a424ed48fcefd0e14916fa1fa983e 100644
--- a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
+++ b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
@@ -9,6 +9,8 @@
state: "latest"
with_items:
- "{{ calico_node_manifests.results }}"
+ - "{{ calico_node_kdd_manifest.results }}"
+ - "{{ calico_node_typha_manifest.results }}"
when:
- inventory_hostname == groups['kube-master'][0] and not item is skipped
loop_control:
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 9084c52248a0ea0d6e788349580ec7dfdcea09ea..5cdd143e352f4f3468dcb4b4291fe11480b5ec19 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -501,3 +501,5 @@ pip_extra_args: |-
etcd_config_dir: /etc/ssl/etcd
etcd_cert_dir: "{{ etcd_config_dir }}/ssl"
+
+typha_enabled: false
diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml
index 3ee4b1b296ec2b2e978391e56dcb27a114be9871..d010bcebea881e6a6e6d4972f818ed7d58e949d2 100644
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@@ -65,3 +65,10 @@ calico_upgrade_needed: false
kube_etcd_cacert_file: ca.pem
kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+
+# Choose data store type for calico: "etcd" or "kdd" (kubernetes datastore)
+calico_datastore: "etcd"
+# Use typha (only with kdd)
+typha_enabled: false
+# Number of typha replicas
+typha_replicas: 1
diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml
index 67b006349b23342da77d13e2d5ccd75baab06b50..321669add149dcdcc28384fdeba86e2f8c7481fb 100644
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@@ -350,3 +350,25 @@
when:
- inventory_hostname in groups['kube-master']
- rbac_enabled or item.type not in rbac_resources
+
+- name: Calico | Create calico manifests for kdd
+ template:
+ src: "{{item.file}}.j2"
+ dest: "{{kube_config_dir}}/{{item.file}}"
+ with_items:
+ - {name: calico, file: kdd-crds.yml, type: kdd}
+ register: calico_node_kdd_manifest
+ when:
+ - inventory_hostname in groups['kube-master']
+ - calico_datastore == "kdd"
+
+- name: Calico | Create calico manifests for typha
+ template:
+ src: "{{item.file}}.j2"
+ dest: "{{kube_config_dir}}/{{item.file}}"
+ with_items:
+ - {name: calico, file: calico-typha.yml, type: typha}
+ register: calico_node_typha_manifest
+ when:
+ - inventory_hostname in groups['kube-master']
+ - typha_enabled and calico_datastore == "kdd"
diff --git a/roles/network_plugin/calico/templates/calico-config.yml.j2 b/roles/network_plugin/calico/templates/calico-config.yml.j2
index dc942d850f429574c6b49cf6d023d44b25b97c26..550e7b7e1172fbba27816bb92ffa8f58c3770e4c 100644
--- a/roles/network_plugin/calico/templates/calico-config.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-config.yml.j2
@@ -4,10 +4,17 @@ metadata:
name: calico-config
namespace: kube-system
data:
+{% if calico_datastore == "etcd" %}
etcd_endpoints: "{{ etcd_access_addresses }}"
etcd_ca: "/calico-secrets/ca_cert.crt"
etcd_cert: "/calico-secrets/cert.crt"
etcd_key: "/calico-secrets/key.pem"
+{% elif calico_datastore == "kdd" and typha_enabled == "true" %}
+ # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas
+ # below. We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is
+ # essential.
+ typha_service_name: "calico-typha"
+{% endif %}
{% if calico_network_backend is defined and calico_network_backend == 'none' %}
cluster_type: "kubespray"
calico_backend: "none"
diff --git a/roles/network_plugin/calico/templates/calico-cr.yml.j2 b/roles/network_plugin/calico/templates/calico-cr.yml.j2
index b32a16745611fd41b994751b11d5334b27a3314d..64d953585287c9f3a656b44b237c872a41e1575d 100644
--- a/roles/network_plugin/calico/templates/calico-cr.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-cr.yml.j2
@@ -19,11 +19,16 @@ rules:
verbs:
- watch
- list
+{% if calico_datastore == "kdd" %}
+ # Used to discover Typhas.
+ - get
+{% endif %}
- apiGroups: [""]
resources:
- nodes/status
verbs:
- patch
+{% if calico_datastore == "etcd" %}
- apiGroups:
- policy
resourceNames:
@@ -32,3 +37,73 @@ rules:
- podsecuritypolicies
verbs:
- use
+{% elif calico_datastore == "kdd" %}
+ # Calico stores some configuration information in node annotations.
+ - update
+ # Watch for changes to Kubernetes NetworkPolicies.
+ - apiGroups: ["networking.k8s.io"]
+ resources:
+ - networkpolicies
+ verbs:
+ - watch
+ - list
+ # Used by Calico for policy information.
+ - apiGroups: [""]
+ resources:
+ - pods
+ - namespaces
+ - serviceaccounts
+ verbs:
+ - list
+ - watch
+ # The CNI plugin patches pods/status.
+ - apiGroups: [""]
+ resources:
+ - pods/status
+ verbs:
+ - patch
+ # Calico monitors various CRDs for config.
+ - apiGroups: ["crd.projectcalico.org"]
+ resources:
+ - globalfelixconfigs
+ - felixconfigurations
+ - bgppeers
+ - globalbgpconfigs
+ - bgpconfigurations
+ - ippools
+ - globalnetworkpolicies
+ - globalnetworksets
+ - networkpolicies
+ - clusterinformations
+ - hostendpoints
+ verbs:
+ - get
+ - list
+ - watch
+ # Calico must create and update some CRDs on startup.
+ - apiGroups: ["crd.projectcalico.org"]
+ resources:
+ - ippools
+ - felixconfigurations
+ - clusterinformations
+ verbs:
+ - create
+ - update
+ # Calico stores some configuration information on the node.
+ - apiGroups: [""]
+ resources:
+ - nodes
+ verbs:
+ - get
+ - list
+ - watch
+ # These permissions are only requried for upgrade from v2.6, and can
+ # be removed after upgrade or on fresh installations.
+ - apiGroups: ["crd.projectcalico.org"]
+ resources:
+ - bgpconfigurations
+ - bgppeers
+ verbs:
+ - create
+ - update
+{% endif %}
diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2
index d8ea78a4486b17076718428d309a2188ccc6821b..cd4841867c71b401bdca89f2c9e620239100515d 100644
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@@ -61,6 +61,13 @@ spec:
# Prevents the container from sleeping forever.
- name: SLEEP
value: "false"
+{% if calico_datastore == "kdd" %}
+ # Set the hostname based on the k8s node name.
+ - name: KUBERNETES_NODE_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+{% endif %}
volumeMounts:
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
@@ -98,11 +105,51 @@ spec:
image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
env:
# The location of the Calico etcd cluster.
+{% if calico_datastore == "etcd" %}
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
+ # Location of the CA certificate for etcd.
+ - name: ETCD_CA_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: calico-config
+ key: etcd_ca
+ # Location of the client key for etcd.
+ - name: ETCD_KEY_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: calico-config
+ key: etcd_key
+ # Location of the client certificate for etcd.
+ - name: ETCD_CERT_FILE
+ valueFrom:
+ configMapKeyRef:
+ name: calico-config
+ key: etcd_cert
+{% elif calico_datastore == "kdd" %}
+ # Use Kubernetes API as the backing datastore.
+ - name: DATASTORE_TYPE
+ value: "kubernetes"
+ {% if typha_enabled == "true" %}
+ # Typha support: controlled by the ConfigMap.
+ - name: FELIX_TYPHAK8SSERVICENAME
+ valueFrom:
+ configMapKeyRef:
+ name: calico-config
+ key: typha_service_name
+ {% endif %}
+ # Wait for the datastore.
+ - name: WAIT_FOR_DATASTORE
+ value: "true"
+ # Set based on the k8s node name.
+ - name: NODENAME
+ valueFrom:
+ fieldRef:
+ fieldPath: spec.nodeName
+{% endif %}
# Choose the backend to use.
- name: CALICO_NETWORKING_BACKEND
valueFrom:
@@ -167,24 +214,6 @@ spec:
- name: CALICO_ADVERTISE_CLUSTER_IPS
value: "{{ kube_service_addresses }}"
{% endif %}
- # Location of the CA certificate for etcd.
- - name: ETCD_CA_CERT_FILE
- valueFrom:
- configMapKeyRef:
- name: calico-config
- key: etcd_ca
- # Location of the client key for etcd.
- - name: ETCD_KEY_FILE
- valueFrom:
- configMapKeyRef:
- name: calico-config
- key: etcd_key
- # Location of the client certificate for etcd.
- - name: ETCD_CERT_FILE
- valueFrom:
- configMapKeyRef:
- name: calico-config
- key: etcd_cert
{% if calico_ip_auto_method is defined %}
- name: IP_AUTODETECTION_METHOD
value: "{{ calico_ip_auto_method }}"
@@ -243,8 +272,10 @@ spec:
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
+{% if calico_datastore == "etcd" %}
- mountPath: /calico-secrets
name: etcd-certs
+{% endif %}
- name: xtables-lock
mountPath: /run/xtables.lock
readOnly: false
@@ -266,10 +297,12 @@ spec:
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
+{% if calico_datastore == "etcd" %}
# Mount in the etcd TLS secrets.
- name: etcd-certs
hostPath:
path: "{{ calico_cert_dir }}"
+{% endif %}
# Mount the global iptables lock file, used by calico/node
- name: xtables-lock
hostPath:
diff --git a/roles/network_plugin/calico/templates/calico-typha.yml.j2 b/roles/network_plugin/calico/templates/calico-typha.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..19e5ec894ab703222902bb2b82025b6e8ca038a3
--- /dev/null
+++ b/roles/network_plugin/calico/templates/calico-typha.yml.j2
@@ -0,0 +1,123 @@
+# This manifest creates a Service, which will be backed by Calico's Typha daemon.
+# Typha sits in between Felix and the API server, reducing Calico's load on the API server.
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: calico-typha
+ namespace: kube-system
+ labels:
+ k8s-app: calico-typha
+spec:
+ ports:
+ - port: 5473
+ protocol: TCP
+ targetPort: calico-typha
+ name: calico-typha
+ selector:
+ k8s-app: calico-typha
+
+---
+
+# This manifest creates a Deployment of Typha to back the above service.
+
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: calico-typha
+ namespace: kube-system
+ labels:
+ k8s-app: calico-typha
+spec:
+ # Number of Typha replicas. To enable Typha, set this to a non-zero value *and* set the
+ # typha_service_name variable in the calico-config ConfigMap above.
+ #
+ # We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is essential
+ # (when using the Kubernetes datastore). Use one replica for every 100-200 nodes. In
+ # production, we recommend running at least 3 replicas to reduce the impact of rolling upgrade.
+ replicas: {{ typha_replicas }}
+ revisionHistoryLimit: 2
+ template:
+ metadata:
+ labels:
+ k8s-app: calico-typha
+ annotations:
+ # This, along with the CriticalAddonsOnly toleration below, marks the pod as a critical
+ # add-on, ensuring it gets priority scheduling and that its resources are reserved
+ # if it ever gets evicted.
+ scheduler.alpha.kubernetes.io/critical-pod: ''
+ cluster-autoscaler.kubernetes.io/safe-to-evict: 'true'
+ spec:
+ nodeSelector:
+ beta.kubernetes.io/os: linux
+ hostNetwork: true
+ tolerations:
+ # Mark the pod as a critical add-on for rescheduling.
+ - key: CriticalAddonsOnly
+ operator: Exists
+ # Since Calico can't network a pod until Typha is up, we need to run Typha itself
+ # as a host-networked pod.
+ serviceAccountName: calico-node
+ containers:
+# - image: calico/typha:v3.4.4
+ - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
+ name: calico-typha
+ ports:
+ - containerPort: 5473
+ name: calico-typha
+ protocol: TCP
+ env:
+ # Enable "info" logging by default. Can be set to "debug" to increase verbosity.
+ - name: TYPHA_LOGSEVERITYSCREEN
+ value: "info"
+ # Disable logging to file and syslog since those don't make sense in Kubernetes.
+ - name: TYPHA_LOGFILEPATH
+ value: "none"
+ - name: TYPHA_LOGSEVERITYSYS
+ value: "none"
+ # Monitor the Kubernetes API to find the number of running instances and rebalance
+ # connections.
+ - name: TYPHA_CONNECTIONREBALANCINGMODE
+ value: "kubernetes"
+ - name: TYPHA_DATASTORETYPE
+ value: "kubernetes"
+ - name: TYPHA_HEALTHENABLED
+ value: "true"
+ # Uncomment these lines to enable prometheus metrics. Since Typha is host-networked,
+ # this opens a port on the host, which may need to be secured.
+ #- name: TYPHA_PROMETHEUSMETRICSENABLED
+ # value: "true"
+ #- name: TYPHA_PROMETHEUSMETRICSPORT
+ # value: "9093"
+ livenessProbe:
+ exec:
+ command:
+ - calico-typha
+ - check
+ - liveness
+ periodSeconds: 30
+ initialDelaySeconds: 30
+ readinessProbe:
+ exec:
+ command:
+ - calico-typha
+ - check
+ - readiness
+ periodSeconds: 10
+
+---
+
+# This manifest creates a Pod Disruption Budget for Typha to allow K8s Cluster Autoscaler to evict
+
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+ name: calico-typha
+ namespace: kube-system
+ labels:
+ k8s-app: calico-typha
+spec:
+ maxUnavailable: 1
+ selector:
+ matchLabels:
+ k8s-app: calico-typha
diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
index be76da0abdcefc0b8f619c7d64fa529881e5d0eb..f9c50e13357ee7d0231b118a4b108567ea131c17 100644
--- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2
+++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
@@ -9,16 +9,25 @@
"nodename": "{{ calico_baremetal_nodename }}",
{% endif %}
"type": "calico",
+ "log_level": "info",
+{% if calico_datastore == "etcd" %}
"etcd_endpoints": "{{ etcd_access_addresses }}",
"etcd_cert_file": "{{ calico_cert_dir }}/cert.crt",
"etcd_key_file": "{{ calico_cert_dir }}/key.pem",
"etcd_ca_cert_file": "{{ calico_cert_dir }}/ca_cert.crt",
- "log_level": "info",
"ipam": {
"type": "calico-ipam",
"assign_ipv4": "true",
"ipv4_pools": ["{{ calico_pool_cidr | default(kube_pods_subnet) }}"]
},
+{% elif calico_datastore == "kdd" %}
+ "datastore_type": "kubernetes",
+ "nodename": "__KUBERNETES_NODE_NAME__",
+ "ipam": {
+ "type": "host-local",
+ "subnet": "usePodCidr"
+ },
+{% endif %}
{% if enable_network_policy %}
"policy": {
"type": "k8s"
diff --git a/roles/network_plugin/calico/templates/kdd-crds.yml.j2 b/roles/network_plugin/calico/templates/kdd-crds.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..3f14729f46cc04d26031531ef446874fa7c50f3b
--- /dev/null
+++ b/roles/network_plugin/calico/templates/kdd-crds.yml.j2
@@ -0,0 +1,133 @@
+# Create all the CustomResourceDefinitions needed for
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: felixconfigurations.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: FelixConfiguration
+ plural: felixconfigurations
+ singular: felixconfiguration
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: bgppeers.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: BGPPeer
+ plural: bgppeers
+ singular: bgppeer
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: bgpconfigurations.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: BGPConfiguration
+ plural: bgpconfigurations
+ singular: bgpconfiguration
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: ippools.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: IPPool
+ plural: ippools
+ singular: ippool
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: hostendpoints.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: HostEndpoint
+ plural: hostendpoints
+ singular: hostendpoint
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: clusterinformations.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: ClusterInformation
+ plural: clusterinformations
+ singular: clusterinformation
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: globalnetworkpolicies.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: GlobalNetworkPolicy
+ plural: globalnetworkpolicies
+ singular: globalnetworkpolicy
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: globalnetworksets.crd.projectcalico.org
+spec:
+ scope: Cluster
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: GlobalNetworkSet
+ plural: globalnetworksets
+ singular: globalnetworkset
+
+---
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+ name: networkpolicies.crd.projectcalico.org
+spec:
+ scope: Namespaced
+ group: crd.projectcalico.org
+ version: v1
+ names:
+ kind: NetworkPolicy
+ plural: networkpolicies
+ singular: networkpolicy