From 23fd3461bcdfd0eafb385bfa0320b682cb8035e0 Mon Sep 17 00:00:00 2001
From: Aivars Sterns <Atoms@users.noreply.github.com>
Date: Thu, 23 Aug 2018 17:17:18 +0300
Subject: [PATCH] calico upgrade to v3 (#3086)
* calico upgrade to v3
* update calico_rr version
* add missing file
* change contents of main.yml as it was left old version
* enable network policy by default
* remove unneeded task
* Fix kubelet calico settings
* fix when statement
* switch back to node-kubeconfig.yaml
---
README.md | 2 +-
docs/calico.md | 9 +-
inventory/sample/group_vars/k8s-cluster.yml | 3 -
roles/download/defaults/main.yml | 10 +-
.../network_plugin/calico/tasks/main.yml | 12 +-
.../policy_controller/calico/tasks/main.yml | 23 +-
.../templates/calico-kube-controllers.yml.j2 | 15 +-
.../calico/templates/calico-kube-cr.yml.j2 | 8 +
roles/kubernetes/kubeadm/tasks/main.yml | 9 +
.../node/templates/kubelet-container.j2 | 1 +
.../node/templates/kubelet.rkt.service.j2 | 7 +
roles/kubernetes/preinstall/tasks/main.yml | 1 +
.../preinstall/tasks/set_resolv_facts.yml | 2 +-
roles/kubespray-defaults/defaults/main.yaml | 2 +-
roles/network_plugin/calico/defaults/main.yml | 11 +-
roles/network_plugin/calico/tasks/check.yml | 37 ++++
roles/network_plugin/calico/tasks/install.yml | 189 +++++++++++++++++
roles/network_plugin/calico/tasks/main.yml | 197 +-----------------
roles/network_plugin/calico/tasks/pre.yml | 16 ++
roles/network_plugin/calico/tasks/upgrade.yml | 26 +++
.../calico/templates/calico-node.yml.j2 | 14 +-
.../calico/templates/cni-calico.conflist.j2 | 10 +-
.../calico/templates/etcdv2-store.yml.j2 | 9 +
.../calico/templates/etcdv3-store.yml.j2 | 9 +
.../canal/templates/canal-node.yaml.j2 | 6 +
roles/reset/tasks/main.yml | 1 +
upgrade-cluster.yml | 22 +-
27 files changed, 399 insertions(+), 252 deletions(-)
create mode 100644 roles/network_plugin/calico/tasks/check.yml
create mode 100644 roles/network_plugin/calico/tasks/install.yml
create mode 100644 roles/network_plugin/calico/tasks/pre.yml
create mode 100644 roles/network_plugin/calico/tasks/upgrade.yml
create mode 100644 roles/network_plugin/calico/templates/etcdv2-store.yml.j2
create mode 100644 roles/network_plugin/calico/templates/etcdv3-store.yml.j2
diff --git a/README.md b/README.md
index 021a5a078..e8475d76b 100644
--- a/README.md
+++ b/README.md
@@ -95,7 +95,7 @@ Supported Components
- [docker](https://www.docker.com/) v17.03 (see note)
- [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2)
- Network Plugin
- - [calico](https://github.com/projectcalico/calico) v2.6.8
+ - [calico](https://github.com/projectcalico/calico) v3.1.3
- [canal](https://github.com/projectcalico/canal) (given calico/flannel versions)
- [cilium](https://github.com/cilium/cilium) v1.2.0
- [contiv](https://github.com/contiv/install) v1.1.7
diff --git a/docs/calico.md b/docs/calico.md
index b8cdc90cb..f6adaa662 100644
--- a/docs/calico.md
+++ b/docs/calico.md
@@ -1,6 +1,13 @@
Calico
===========
+---
+ **N.B. Version 2.6.5 upgrade to 3.1.1 is upgrading etcd store to etcdv3**
+ If you create automated backups of etcdv2 please switch for creating etcdv3 backups, as kubernetes and calico now uses etcdv3
+ After migration you can check `/tmp/calico_upgrade/` directory for converted items to etcdv3.
+ **PLEASE TEST upgrade before upgrading production cluster.**
+ ---
+
Check if the calico-node container is running
```
@@ -86,7 +93,7 @@ To do so you can deploy BGP route reflectors and peer `calico-node` with them as
recommended here:
* https://hub.docker.com/r/calico/routereflector/
-* http://docs.projectcalico.org/v2.0/reference/private-cloud/l3-interconnect-fabric
+* https://docs.projectcalico.org/v3.1/reference/private-cloud/l3-interconnect-fabric
You need to edit your inventory and add:
diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml
index ded955066..119d5af09 100644
--- a/inventory/sample/group_vars/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster.yml
@@ -83,9 +83,6 @@ kube_network_plugin: calico
# weave_no_masq_local: true
# weave_extra_args: ~
-# Enable kubernetes network policies
-enable_network_policy: false
-
# Kubernetes internal network for services, unused block of space.
kube_service_addresses: 10.233.0.0/18
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index c726b40bd..97c8d8562 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -32,11 +32,11 @@ kubeadm_version: "{{ kube_version }}"
etcd_version: v3.2.18
# TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
# after migration to container download
-calico_version: "v2.6.8"
-calico_ctl_version: "v1.6.3"
-calico_cni_version: "v1.11.4"
-calico_policy_version: "v1.0.3"
-calico_rr_version: "v0.4.2"
+calico_version: "v3.1.3"
+calico_ctl_version: "v3.1.3"
+calico_cni_version: "v3.1.3"
+calico_policy_version: "v3.1.3"
+calico_rr_version: "v0.6.1"
flannel_version: "v0.10.0"
flannel_cni_version: "v0.3.0"
vault_version: 0.10.1
diff --git a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
index 4c8295c1e..013d3b056 100644
--- a/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
+++ b/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml
@@ -7,5 +7,13 @@
resource: "{{item.item.type}}"
filename: "{{kube_config_dir}}/{{item.item.file}}"
state: "latest"
- with_items: "{{ calico_node_manifests.results }}"
- when: inventory_hostname == groups['kube-master'][0] and not item|skipped
+ with_items:
+ - "{{ calico_node_manifests.results }}"
+ when:
+ - inventory_hostname == groups['kube-master'][0] and not item|skipped
+
+- name: "calico upgrade complete"
+ shell: "{{ bin_dir }}/calico-upgrade complete --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml"
+ when:
+ - inventory_hostname == groups['kube-master'][0]
+ - calico_upgrade_needed|default(False)
diff --git a/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
index bc2fdf7e7..7e9377da4 100644
--- a/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
+++ b/roles/kubernetes-apps/policy_controller/calico/tasks/main.yml
@@ -2,20 +2,12 @@
- name: Set cert dir
set_fact:
calico_cert_dir: "{{ canal_cert_dir }}"
- when: kube_network_plugin == 'canal'
+ when:
+ - kube_network_plugin == 'canal'
tags:
- facts
- canal
-- name: Delete the old calico-policy-controller if it exist
- kube:
- name: calico-policy-controller
- kubectl: "{{bin_dir}}/kubectl"
- resource: rs
- namespace: "kube-system"
- state: absent
- run_once: true
-
- name: Create calico-kube-controllers manifests
template:
src: "{{item.file}}.j2"
@@ -26,7 +18,9 @@
- {name: calico-kube-controllers, file: calico-kube-cr.yml, type: clusterrole}
- {name: calico-kube-controllers, file: calico-kube-crb.yml, type: clusterrolebinding}
register: calico_kube_manifests
- when: inventory_hostname == groups['kube-master'][0] and not item|skipped
+ when:
+ - inventory_hostname == groups['kube-master'][0]
+ - rbac_enabled or item.type not in rbac_resources
- name: Start of Calico kube controllers
kube:
@@ -36,5 +30,8 @@
resource: "{{item.item.type}}"
filename: "{{kube_config_dir}}/{{item.item.file}}"
state: "latest"
- with_items: "{{ calico_kube_manifests.results }}"
- when: inventory_hostname == groups['kube-master'][0] and not item|skipped
+ with_items:
+ - "{{ calico_kube_manifests.results }}"
+ when:
+ - inventory_hostname == groups['kube-master'][0]
+ - not item|skipped
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
index 5dcafbbc5..d99466248 100644
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2
@@ -6,8 +6,12 @@ metadata:
labels:
k8s-app: calico-kube-controllers
kubernetes.io/cluster-service: "true"
+ annotations:
+ scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
replicas: 1
+ strategy:
+ type: Recreate
selector:
matchLabels:
kubernetes.io/cluster-service: "true"
@@ -45,17 +49,6 @@ spec:
value: "{{ calico_cert_dir }}/cert.crt"
- name: ETCD_KEY_FILE
value: "{{ calico_cert_dir }}/key.pem"
- # Location of the Kubernetes API - this shouldn't need to be
- # changed so long as it is used in conjunction with
- # CONFIGURE_ETC_HOSTS="true".
- - name: K8S_API
- value: "https://kubernetes.default"
- # Configure /etc/hosts within the container to resolve
- # the kubernetes.default Service to the correct clusterIP
- # using the environment provided by the kubelet.
- # This removes the need for KubeDNS to resolve the Service.
- - name: CONFIGURE_ETC_HOSTS
- value: "true"
volumeMounts:
- mountPath: {{ calico_cert_dir }}
name: etcd-certs
diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2
index d05e986a4..bf582c769 100644
--- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2
+++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-cr.yml.j2
@@ -12,6 +12,14 @@ rules:
- pods
- namespaces
- networkpolicies
+ - nodes
verbs:
- watch
- list
+ - apiGroups:
+ - networking.k8s.io
+ resources:
+ - networkpolicies
+ verbs:
+ - watch
+ - list
\ No newline at end of file
diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml
index c0d436b66..5697076e8 100644
--- a/roles/kubernetes/kubeadm/tasks/main.yml
+++ b/roles/kubernetes/kubeadm/tasks/main.yml
@@ -88,6 +88,15 @@
- is_kube_master
- kubeadm_discovery_address != kube_apiserver_endpoint
+# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
+- name: Symlink kubelet kubeconfig for calico/canal
+ file:
+ src: "{{ kube_config_dir }}/kubelet.conf"
+ dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+ state: link
+ force: yes
+ when: kube_network_plugin in ['calico','canal']
+
- name: Restart all kube-proxy pods to ensure that they load the new configmap
shell: "{{ bin_dir }}/kubectl delete pod -n kube-system -l k8s-app=kube-proxy"
delegate_to: "{{groups['kube-master']|first}}"
diff --git a/roles/kubernetes/node/templates/kubelet-container.j2 b/roles/kubernetes/node/templates/kubelet-container.j2
index dcf86c327..58e54560d 100644
--- a/roles/kubernetes/node/templates/kubelet-container.j2
+++ b/roles/kubernetes/node/templates/kubelet-container.j2
@@ -22,6 +22,7 @@
-v {{ docker_daemon_graph }}:{{ docker_daemon_graph }}:rw \
-v /var/log:/var/log:rw \
-v /var/lib/kubelet:/var/lib/kubelet:shared \
+ -v /var/lib/calico:/var/lib/calico:shared \
-v /var/lib/cni:/var/lib/cni:shared \
-v /var/run:/var/run:rw \
{# we can run into issues with double mounting /var/lib/kubelet #}
diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2
index d75f4c662..46557d273 100644
--- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2
+++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2
@@ -37,6 +37,10 @@ ExecStart=/usr/bin/rkt run \
--volume etc-cni,kind=host,source=/etc/cni,readOnly=true \
--volume opt-cni,kind=host,source=/opt/cni,readOnly=true \
--volume var-lib-cni,kind=host,source=/var/lib/cni,readOnly=false \
+{% endif %}
+{% if kube_network_plugin in ["calico", "canal"] %}
+ --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=false \
+{% endif %}
{# we can run into issues with double mounting /var/lib/kubelet #}
{# surely there's a better way to do this #}
{% if '/var/lib/kubelet' not in kubelet_flexvolumes_plugins_dir %}
@@ -55,6 +59,9 @@ ExecStart=/usr/bin/rkt run \
--mount volume=etc-cni,target=/etc/cni \
--mount volume=opt-cni,target=/opt/cni \
--mount volume=var-lib-cni,target=/var/lib/cni \
+{% if kube_network_plugin in ["calico", "canal"] %}
+ --mount volume=var-lib-calico,target=/var/lib/calico \
+{% endif %}
{% endif %}
--mount volume=os-release,target=/etc/os-release \
--mount volume=dns,target=/etc/resolv.conf \
diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml
index 4db366ced..83347b59f 100644
--- a/roles/kubernetes/preinstall/tasks/main.yml
+++ b/roles/kubernetes/preinstall/tasks/main.yml
@@ -89,6 +89,7 @@
with_items:
- "/etc/cni/net.d"
- "/opt/cni/bin"
+ - "/var/lib/calico"
when:
- kube_network_plugin in ["calico", "weave", "canal", "flannel", "contiv", "cilium"]
- inventory_hostname in groups['k8s-cluster']
diff --git a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
index eb8f3f43f..4e535fb0f 100644
--- a/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
+++ b/roles/kubernetes/preinstall/tasks/set_resolv_facts.yml
@@ -98,7 +98,7 @@
{%- elif dns_mode == 'coredns_dual' and not dns_early|bool -%}
{{ [ skydns_server ] + [ skydns_server_secondary ] + upstream_dns_servers|default([]) }}
{%- elif dns_mode == 'manual' and not dns_early|bool -%}
- {{ [ manual_dns_server ] + upstream_dns_servers|default([]) }}
+ {{ ( manual_dns_server.split(',') | list) + upstream_dns_servers|default([]) }}
{%- elif dns_early|bool -%}
{{ upstream_dns_servers|default([]) }}
{%- else -%}
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 888cbc3c8..4808daf16 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -187,7 +187,7 @@ dashboard_enabled: true
efk_enabled: false
helm_enabled: false
registry_enabled: false
-enable_network_policy: false
+enable_network_policy: true
local_volume_provisioner_enabled: "{{ local_volumes_enabled | default('false') }}"
persistent_volumes_enabled: false
cephfs_provisioner_enabled: false
diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml
index 800bbb6fc..e65d569ed 100644
--- a/roles/network_plugin/calico/defaults/main.yml
+++ b/roles/network_plugin/calico/defaults/main.yml
@@ -2,9 +2,12 @@
# Enables Internet connectivity from containers
nat_outgoing: true
+#add default ippool name
+calico_pool_name: "default-pool"
+
# Use IP-over-IP encapsulation across hosts
ipip: true
-ipip_mode: always # change to "cross-subnet" if you only want ipip encapsulation on traffic going across subnets
+ipip_mode: Always # change to "CrossSubnet" if you only want ipip encapsulation on traffic going across subnets
# Set to true if you want your calico cni binaries to overwrite the
# ones from hyperkube while leaving other cni plugins intact.
@@ -37,6 +40,9 @@ calico_felix_prometheusmetricsport: 9091
calico_felix_prometheusgometricsenabled: "true"
calico_felix_prometheusprocessmetricsenabled: "true"
+### check latest version https://github.com/projectcalico/calico-upgrade/releases
+calico_upgrade_version: v1.0.5
+
# Should calico ignore kernel's RPF check setting,
# see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198
calico_node_ignorelooserpf: false
@@ -48,3 +54,6 @@ calico_node_ignorelooserpf: false
# calico_ip_auto_method: "interface=eth.*"
calico_baremetal_nodename: "{{ inventory_hostname }}"
+
+### do not enable this, this is detected in scope of tasks, this is just a default value
+calico_upgrade_needed: false
\ No newline at end of file
diff --git a/roles/network_plugin/calico/tasks/check.yml b/roles/network_plugin/calico/tasks/check.yml
new file mode 100644
index 000000000..4eca9cc65
--- /dev/null
+++ b/roles/network_plugin/calico/tasks/check.yml
@@ -0,0 +1,37 @@
+---
+- name: "Check vars defined correctly"
+ assert:
+ that:
+ - "calico_pool_name is defined"
+ - "calico_pool_name | match('^[a-zA-Z0-9-_\\\\.]{2,63}$')"
+ - "ipip_mode is defined"
+ - "ipip_mode in ['Always', 'CrossSubnet', 'Never']"
+ msg: "Check variable definitions seems something is wrong"
+ run_once: yes
+
+- name: "Get current version of calico cluster version"
+ shell: "{{ bin_dir }}/calicoctl version | grep 'Cluster Version' | awk '{ print $3}'"
+ register: calico_version_on_server
+ run_once: yes
+ delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: "Determine if calico upgrade is needed"
+ block:
+ - name: "Check that calico version is enought for upgrade"
+ assert:
+ that:
+ - calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
+ msg: "Your version of calico is not fresh enough for upgrade"
+
+ - name: "Set upgrade flag when version needs to be updated"
+ set_fact:
+ calico_upgrade_needed: True
+ when:
+ - calico_version_on_server.stdout|version_compare('v2.6.5', '>=')
+ - calico_version_on_server.stdout|version_compare('v3.0.0', '<')
+
+ when:
+ - 'calico_version_on_server.stdout is defined'
+ - 'calico_version_on_server.stdout != ""'
+ - inventory_hostname == groups['kube-master'][0]
+ run_once: yes
\ No newline at end of file
diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml
new file mode 100644
index 000000000..d658d575e
--- /dev/null
+++ b/roles/network_plugin/calico/tasks/install.yml
@@ -0,0 +1,189 @@
+---
+
+- name: Calico | Write Calico cni config
+ template:
+ src: "cni-calico.conflist.j2"
+ dest: "/etc/cni/net.d/10-calico.conflist"
+ owner: kube
+
+- name: Calico | Create calico certs directory
+ file:
+ dest: "{{ calico_cert_dir }}"
+ state: directory
+ mode: 0750
+ owner: root
+ group: root
+
+- name: Calico | Link etcd certificates for calico-node
+ file:
+ src: "{{ etcd_cert_dir }}/{{ item.s }}"
+ dest: "{{ calico_cert_dir }}/{{ item.d }}"
+ state: hard
+ force: yes
+ with_items:
+ - {s: "ca.pem", d: "ca_cert.crt"}
+ - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
+ - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
+
+- name: Calico | Install calicoctl container script
+ template:
+ src: calicoctl-container.j2
+ dest: "{{ bin_dir }}/calicoctl"
+ mode: 0755
+ owner: root
+ group: root
+ changed_when: false
+
+- name: Calico | Copy cni plugins from hyperkube
+ command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -r /opt/cni/bin/. /cnibindir/"
+ register: cni_task_result
+ until: cni_task_result.rc == 0
+ retries: 4
+ delay: "{{ retry_stagger | random + 3 }}"
+ changed_when: false
+ tags:
+ - hyperkube
+ - upgrade
+
+- name: Calico | Copy cni plugins from calico/cni container
+ command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp /opt/cni/bin/* /cnibindir/'"
+ register: cni_task_result
+ until: cni_task_result.rc == 0
+ retries: 4
+ delay: "{{ retry_stagger | random + 3 }}"
+ changed_when: false
+ when:
+ - "overwrite_hyperkube_cni|bool"
+ tags:
+ - hyperkube
+ - upgrade
+
+- name: Calico | Set cni directory permissions
+ file:
+ path: /opt/cni/bin
+ state: directory
+ owner: kube
+ recurse: true
+ mode: 0755
+
+- name: Calico | wait for etcd
+ uri:
+ url: "{{ etcd_access_addresses.split(',') | first }}/health"
+ validate_certs: no
+ client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+ client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+ register: result
+ until: result.status == 200 or result.status == 401
+ retries: 10
+ delay: 5
+ run_once: true
+
+- name: Calico | Check if calico network pool has already been configured
+ shell: >
+ {{ bin_dir }}/calicoctl get ippool | grep -w "{{ kube_pods_subnet }}" | wc -l
+ register: calico_conf
+ retries: 4
+ delay: "{{ retry_stagger | random + 3 }}"
+ delegate_to: "{{ groups['kube-master'][0] }}"
+ run_once: true
+
+- name: Calico | Configure calico network pool
+ shell: >
+ echo "
+ { "kind": "IPPool",
+ "apiVersion": "projectcalico.org/v3",
+ "metadata": {
+ "name": "{{ calico_pool_name }}",
+ },
+ "spec": {
+ "cidr": "{{ kube_pods_subnet }}",
+ "ipipMode": "{{ ipip_mode|capitalize }}",
+ "natOutgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }} }} " | {{ bin_dir }}/calicoctl create -f -
+ run_once: true
+ delegate_to: "{{ groups['kube-master'][0] }}"
+ when:
+ - 'calico_conf.stdout == "0"'
+
+- name: "Determine nodeToNodeMesh needed state"
+ set_fact:
+ nodeToNodeMeshEnabled: "false"
+ when:
+ - peer_with_router|default(false) or peer_with_calico_rr|default(false)
+ - inventory_hostname in groups['k8s-cluster']
+ run_once: yes
+
+
+- name: Calico | Set global as_num
+ shell: >
+ echo '
+ { "kind": "BGPConfiguration",
+ "apiVersion": "projectcalico.org/v3",
+ "metadata": {
+ "name": "default",
+ },
+ "spec": {
+ "logSeverityScreen": "Info",
+ "nodeToNodeMeshEnabled": {{ nodeToNodeMeshEnabled|default('true') }} ,
+ "asNumber": {{ global_as_num }} }} ' | {{ bin_dir }}/calicoctl --skip-exists create -f -
+ run_once: true
+ delegate_to: "{{ groups['kube-master'][0] }}"
+
+- name: Calico | Configure peering with router(s)
+ shell: >
+ echo '{
+ "apiVersion": "projectcalico.org/v3",
+ "kind": "bgpPeer",
+ "metadata": {
+ "name": "{{ inventory_hostname }}-bgp"
+ },
+ "spec": {
+ "asNumber": "{{ item.as }}",
+ "node": "{{ inventory_hostname }}",
+ "scope": "node",
+ "peerIP": "{{ item.router_id }}"
+ }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+ retries: 4
+ delay: "{{ retry_stagger | random + 3 }}"
+ with_items:
+ - "{{ peers|default([]) }}"
+ when:
+ - peer_with_router|default(false)
+ - inventory_hostname in groups['k8s-cluster']
+
+- name: Calico | Configure peering with route reflectors
+ shell: >
+ echo '{
+ "apiVersion": "projectcalico.org/v3",
+ "kind": "bgpPeer",
+ "metadata": {
+ "name": "{{ inventory_hostname }}"
+ },
+ "spec": {
+ "asNumber": "{{ local_as | default(global_as_num)}}",
+ "scope": "node",
+ "node": "{{ inventory_hostname }}",
+ "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"
+ }}' | {{ bin_dir }}/calicoctl create --skip-exists -f -
+ retries: 4
+ delay: "{{ retry_stagger | random + 3 }}"
+ with_items:
+ - "{{ groups['calico-rr'] | default([]) }}"
+ when:
+ - (peer_with_calico_rr|default(false)
+ - inventory_hostname in groups['k8s-cluster']
+ - hostvars[item]['cluster_id'] == cluster_id)
+
+- name: Calico | Create calico manifests
+ template:
+ src: "{{item.file}}.j2"
+ dest: "{{kube_config_dir}}/{{item.file}}"
+ with_items:
+ - {name: calico-config, file: calico-config.yml, type: cm}
+ - {name: calico-node, file: calico-node.yml, type: ds}
+ - {name: calico, file: calico-node-sa.yml, type: sa}
+ - {name: calico, file: calico-cr.yml, type: clusterrole}
+ - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+ register: calico_node_manifests
+ when:
+ - inventory_hostname in groups['kube-master']
+ - rbac_enabled or item.type not in rbac_resources
\ No newline at end of file
diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml
index 7ce84cfdc..72d8e3e11 100644
--- a/roles/network_plugin/calico/tasks/main.yml
+++ b/roles/network_plugin/calico/tasks/main.yml
@@ -1,193 +1,12 @@
---
-- name: Calico | Disable calico-node service if it exists
- service:
- name: calico-node
- state: stopped
- enabled: yes
- failed_when: false
+- include_tasks: check.yml
-- name: Calico | Get kubelet hostname
- shell: >-
- {{ bin_dir }}/kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address'
- | egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1
- register: calico_kubelet_name
- delegate_to: "{{ groups['kube-master'][0] }}"
- when: cloud_provider is defined
-
-- name: Calico | Write Calico cni config
- template:
- src: "cni-calico.conflist.j2"
- dest: "/etc/cni/net.d/10-calico.conflist"
- owner: kube
-
-- name: Calico | Create calico certs directory
- file:
- dest: "{{ calico_cert_dir }}"
- state: directory
- mode: 0750
- owner: root
- group: root
-
-- name: Calico | Link etcd certificates for calico-node
- file:
- src: "{{ etcd_cert_dir }}/{{ item.s }}"
- dest: "{{ calico_cert_dir }}/{{ item.d }}"
- state: hard
- force: yes
- with_items:
- - {s: "ca.pem", d: "ca_cert.crt"}
- - {s: "node-{{ inventory_hostname }}.pem", d: "cert.crt"}
- - {s: "node-{{ inventory_hostname }}-key.pem", d: "key.pem"}
-
-- name: Calico | Install calicoctl container script
- template:
- src: calicoctl-container.j2
- dest: "{{ bin_dir }}/calicoctl"
- mode: 0755
- owner: root
- group: root
- changed_when: false
-
-- name: Calico | Copy cni plugins from hyperkube
- command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -rf /opt/cni/bin/. /cnibindir/"
- register: cni_task_result
- until: cni_task_result.rc == 0
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- changed_when: false
- tags:
- - hyperkube
- - upgrade
-
-- name: Calico | Copy cni plugins from calico/cni container
- command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ calico_cni_image_repo }}:{{ calico_cni_image_tag }} sh -c 'cp /opt/cni/bin/* /cnibindir/'"
- register: cni_task_result
- until: cni_task_result.rc == 0
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- changed_when: false
- when: overwrite_hyperkube_cni|bool
- tags:
- - hyperkube
- - upgrade
-
-- name: Calico | Set cni directory permissions
- file:
- path: /opt/cni/bin
- state: directory
- owner: kube
- recurse: true
- mode: 0755
-
-- name: Calico | wait for etcd
- uri:
- url: "{{ etcd_access_addresses.split(',') | first }}/health"
- validate_certs: no
- client_cert: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
- client_key: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
- register: result
- until: result.status == 200 or result.status == 401
- retries: 10
- delay: 5
- run_once: true
-
-- name: Calico | Check if calico network pool has already been configured
- command: |-
- curl \
- --cacert {{ etcd_cert_dir }}/ca.pem \
- --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
- --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
- {{ etcd_access_addresses.split(',') | first }}/v2/keys/calico/v1/ipam/v4/pool
- register: calico_conf
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- run_once: true
- changed_when: false
+- include_tasks: pre.yml
-- name: Calico | Configure calico network pool
- shell: >
- echo '{
- "kind": "ipPool",
- "spec": {"disabled": false, "ipip": {"enabled": {{ ipip }}, "mode": "{{ ipip_mode }}"},
- "nat-outgoing": {{ nat_outgoing|default(false) and not peer_with_router|default(false) }}},
- "apiVersion": "v1",
- "metadata": {"cidr": "{{ kube_pods_subnet }}"}
- }'
- | {{ bin_dir }}/calicoctl apply -f -
- environment:
- NO_DEFAULT_POOLS: true
- run_once: true
- when: ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
-
-- name: Calico | Get calico configuration from etcd
- command: |-
- curl \
- --cacert {{ etcd_cert_dir }}/ca.pem \
- --cert {{ etcd_cert_dir}}/node-{{ inventory_hostname }}.pem \
- --key {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem \
- {{ etcd_access_addresses.split(',') | first }}/v2/keys/calico/v1/ipam/v4/pool
- register: calico_pools_raw
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- run_once: true
-
-- set_fact:
- calico_pools: "{{ calico_pools_raw.stdout | from_json }}"
- run_once: true
-
-- name: Calico | Set global as_num
- command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
- run_once: true
-
-- name: Calico | Disable node mesh
- shell: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
- and inventory_hostname in groups['k8s-cluster'])
- run_once: true
-
-- name: Calico | Configure peering with router(s)
- shell: >
- echo '{
- "kind": "bgpPeer",
- "spec": {"asNumber": "{{ item.as }}"},
- "apiVersion": "v1",
- "metadata": {"node": "{{ inventory_hostname }}", "scope": "node", "peerIP": "{{ item.router_id }}"}
- }'
- | {{ bin_dir }}/calicoctl create --skip-exists -f -
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- with_items: "{{ peers|default([]) }}"
- when: peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster']
-
-- name: Calico | Configure peering with route reflectors
- shell: >
- echo '{
- "kind": "bgpPeer",
- "spec": {"asNumber": "{{ local_as | default(global_as_num)}}"},
- "apiVersion": "v1",
- "metadata": {"node": "{{ inventory_hostname }}",
- "scope": "node",
- "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(hostvars[item]["ansible_default_ipv4"]["address"]) }}"}
- }'
- | {{ bin_dir }}/calicoctl create --skip-exists -f -
- retries: 4
- delay: "{{ retry_stagger | random + 3 }}"
- with_items: "{{ groups['calico-rr'] | default([]) }}"
- when: (peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
- and hostvars[item]['cluster_id'] == cluster_id)
-
-- name: Calico | Create calico manifests
- template:
- src: "{{item.file}}.j2"
- dest: "{{kube_config_dir}}/{{item.file}}"
- with_items:
- - {name: calico-config, file: calico-config.yml, type: cm}
- - {name: calico-node, file: calico-node.yml, type: ds}
- - {name: calico, file: calico-node-sa.yml, type: sa}
- - {name: calico, file: calico-cr.yml, type: clusterrole}
- - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
- register: calico_node_manifests
+- include_tasks: upgrade.yml
when:
- - inventory_hostname in groups['kube-master']
+ - calico_upgrade_needed
+ run_once: yes
+ delegate_to: "{{ groups['kube-master'][0] }}"
+
+- include_tasks: install.yml
diff --git a/roles/network_plugin/calico/tasks/pre.yml b/roles/network_plugin/calico/tasks/pre.yml
new file mode 100644
index 000000000..4781541bd
--- /dev/null
+++ b/roles/network_plugin/calico/tasks/pre.yml
@@ -0,0 +1,16 @@
+---
+- name: Calico | Disable calico-node service if it exists
+ service:
+ name: calico-node
+ state: stopped
+ enabled: no
+ failed_when: false
+
+- name: Calico | Get kubelet hostname
+ shell: >-
+ {{ bin_dir }}/kubectl get node -o custom-columns='NAME:.metadata.name,INTERNAL-IP:.status.addresses[?(@.type=="InternalIP")].address'
+ | egrep "{{ ansible_all_ipv4_addresses | join('$|') }}$" | cut -d" " -f1
+ register: calico_kubelet_name
+ delegate_to: "{{ groups['kube-master'][0] }}"
+ when:
+ - "cloud_provider is defined"
\ No newline at end of file
diff --git a/roles/network_plugin/calico/tasks/upgrade.yml b/roles/network_plugin/calico/tasks/upgrade.yml
new file mode 100644
index 000000000..443362c98
--- /dev/null
+++ b/roles/network_plugin/calico/tasks/upgrade.yml
@@ -0,0 +1,26 @@
+---
+- name: "Download calico-upgrade tool (force version)"
+ get_url:
+ url: "https://github.com/projectcalico/calico-upgrade/releases/download/{{ calico_upgrade_version }}/calico-upgrade"
+ dest: "{{ bin_dir }}/calico-upgrade"
+ mode: 0755
+ owner: root
+ group: root
+ force: yes
+
+- name: "Create etcdv2 and etcdv3 calicoApiConfig"
+ template:
+ src: "{{ item }}-store.yml.j2"
+ dest: "/etc/calico/{{ item }}.yml"
+ with_items:
+ - "etcdv2"
+ - "etcdv3"
+
+- name: "Tests data migration (dry-run)"
+ shell: "{{ bin_dir }}/calico-upgrade dry-run --output-dir=/tmp --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml"
+ register: calico_upgrade_test_data
+ failed_when: '"Successfully" not in calico_upgrade_test_data.stdout'
+
+- name: "If test migration is success continue with calico data real migration"
+ shell: "{{ bin_dir }}/calico-upgrade start --no-prompts --apiconfigv1 /etc/calico/etcdv2.yml --apiconfigv3 /etc/calico/etcdv3.yml --output-dir=/tmp/calico_upgrade"
+ register: calico_upgrade_migration_data
\ No newline at end of file
diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2
index 54dacba8f..38469624e 100644
--- a/roles/network_plugin/calico/templates/calico-node.yml.j2
+++ b/roles/network_plugin/calico/templates/calico-node.yml.j2
@@ -77,12 +77,6 @@ spec:
# Set Felix logging to "info"
- name: FELIX_LOGSEVERITYSCREEN
value: "info"
- # Disable autocreation of pools
- - name: CALICO_NO_DEFAULT_POOLS
- value: "true"
- # Enable libnetwork
- - name: CALICO_LIBNETWORK_ENABLED
- value: "true"
# Set MTU for tunnel device used if ipip is enabled
{% if calico_mtu is defined %}
- name: FELIX_IPINIPMTU
@@ -158,6 +152,8 @@ spec:
readOnly: true
- mountPath: /var/run/calico
name: var-run-calico
+ - mountPath: /var/lib/calico
+ name: var-lib-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
@@ -169,6 +165,9 @@ spec:
- name: var-run-calico
hostPath:
path: /var/run/calico
+ - name: var-lib-calico
+ hostPath:
+ path: /var/lib/calico
# Used to install CNI.
- name: cni-bin-dir
hostPath:
@@ -183,5 +182,4 @@ spec:
updateStrategy:
rollingUpdate:
maxUnavailable: {{ serial | default('20%') }}
- type: RollingUpdate
-
+ type: RollingUpdate
\ No newline at end of file
diff --git a/roles/network_plugin/calico/templates/cni-calico.conflist.j2 b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
index 443e3b43b..e84ab7d61 100644
--- a/roles/network_plugin/calico/templates/cni-calico.conflist.j2
+++ b/roles/network_plugin/calico/templates/cni-calico.conflist.j2
@@ -3,7 +3,7 @@
"cniVersion":"0.3.1",
"plugins":[
{
- {% if cloud_provider is defined %}
+{% if cloud_provider is defined %}
"nodename": "{{ calico_kubelet_name.stdout }}",
{% else %}
"nodename": "{{ calico_baremetal_nodename }}",
@@ -19,14 +19,14 @@
"assign_ipv4": "true",
"ipv4_pools": ["{{ kube_pods_subnet }}"]
},
- {% if enable_network_policy %}
+{% if enable_network_policy %}
"policy": {
"type": "k8s"
},
- {%- endif %}
- {% if calico_mtu is defined and calico_mtu is number %}
+{%- endif %}
+{% if calico_mtu is defined and calico_mtu is number %}
"mtu": {{ calico_mtu }},
- {%- endif %}
+{%- endif %}
"kubernetes": {
"kubeconfig": "{{ kube_config_dir }}/node-kubeconfig.yaml"
}
diff --git a/roles/network_plugin/calico/templates/etcdv2-store.yml.j2 b/roles/network_plugin/calico/templates/etcdv2-store.yml.j2
new file mode 100644
index 000000000..c65728838
--- /dev/null
+++ b/roles/network_plugin/calico/templates/etcdv2-store.yml.j2
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: calicoApiConfig
+metadata:
+spec:
+ datastoreType: "etcdv2"
+ etcdEndpoints: "{{ etcd_access_addresses }}"
+ etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+ etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+ etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem"
diff --git a/roles/network_plugin/calico/templates/etcdv3-store.yml.j2 b/roles/network_plugin/calico/templates/etcdv3-store.yml.j2
new file mode 100644
index 000000000..3dc566d0f
--- /dev/null
+++ b/roles/network_plugin/calico/templates/etcdv3-store.yml.j2
@@ -0,0 +1,9 @@
+apiVersion: projectcalico.org/v3
+kind: CalicoAPIConfig
+metadata:
+spec:
+ datastoreType: "etcdv3"
+ etcdEndpoints: "{{ etcd_access_addresses }}"
+ etcdKeyFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem"
+ etcdCertFile: "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem"
+ etcdCACertFile: "{{ etcd_cert_dir }}/ca.pem"
\ No newline at end of file
diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2
index 7e3ecf8d5..6ef251af8 100644
--- a/roles/network_plugin/canal/templates/canal-node.yaml.j2
+++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2
@@ -28,6 +28,9 @@ spec:
- name: lib-modules
hostPath:
path: /lib/modules
+ - name: var-lib-calico
+ hostPath:
+ path: /var/lib/calico
- name: var-run-calico
hostPath:
path: /var/run/calico
@@ -202,6 +205,9 @@ spec:
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
+ - mountPath: /var/lib/calico
+ name: var-lib-calico
+ readOnly: false
- name: "canal-certs"
mountPath: "{{ canal_cert_dir }}"
readOnly: true
diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml
index fe72c2bec..47b51546f 100644
--- a/roles/reset/tasks/main.yml
+++ b/roles/reset/tasks/main.yml
@@ -136,6 +136,7 @@
- "{{ bin_dir }}/kubeadm"
- "{{ bin_dir }}/helm"
- "{{ bin_dir }}/calicoctl"
+ - "{{ bin_dir }}/calico-upgrade"
- "{{ bin_dir }}/weave"
- /var/lib/rkt
- /etc/vault
diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml
index 3d75883dd..abc89e18f 100644
--- a/upgrade-cluster.yml
+++ b/upgrade-cluster.yml
@@ -78,10 +78,19 @@
- { role: kubernetes/master, tags: master }
- { role: kubernetes/client, tags: client }
- { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
- - { role: network_plugin, tags: network }
- { role: upgrade/post-upgrade, tags: post-upgrade }
environment: "{{proxy_env}}"
+#Upgrade calico on all masters and nodes
+- hosts: kube-master:kube-node
+ any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+ serial: "{{ serial | default('20%') }}"
+ roles:
+ - { role: kubespray-defaults}
+ - { role: network_plugin, tags: network }
+ - { role: kubernetes-apps/network_plugin, tags: network }
+ - { role: kubernetes-apps/policy_controller, tags: policy-controller }
+
#Finally handle worker upgrades, based on given batch size
- hosts: kube-node:!kube-master
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
@@ -90,7 +99,6 @@
- { role: kubespray-defaults}
- { role: upgrade/pre-upgrade, tags: pre-upgrade }
- { role: kubernetes/node, tags: node }
- - { role: network_plugin, tags: network }
- { role: upgrade/post-upgrade, tags: post-upgrade }
- { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
- { role: kubespray-defaults}
@@ -102,14 +110,6 @@
- { role: kubespray-defaults}
- { role: kubernetes-apps/rotate_tokens, tags: rotate_tokens, when: "secret_changed|default(false)" }
-- hosts: kube-master
- any_errors_fatal: true
- roles:
- - { role: kubespray-defaults}
- - { role: kubernetes-apps/network_plugin, tags: network }
- - { role: kubernetes-apps/policy_controller, tags: policy-controller }
- - { role: kubernetes/client, tags: client }
-
- hosts: calico-rr
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles:
@@ -127,4 +127,4 @@
any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
roles:
- { role: kubespray-defaults}
- - { role: kubernetes-apps, tags: apps }
+ - { role: kubernetes-apps, tags: apps }
\ No newline at end of file
--
GitLab