Move calico to daemonset (#1605)

* Drop legacy calico logic

* add calico as a daemonset

.gitlab-ci.yml

@@ -259,6 +259,7 @@ before_script:
 # Test matrix. Leave the comments for markup scripts.
 .coreos_calico_sep_variables: &coreos_calico_sep_variables
 # stage: deploy-gce-part1
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
   KUBE_NETWORK_PLUGIN: calico
   CLOUD_IMAGE: coreos-stable-1465-6-0-v20170817
   CLOUD_REGION: us-west1-b

roles/kubernetes-apps/network_plugin/calico/tasks/main.yml

+---
+- name: Start Calico resources
+  kube:
+    name: "{{item.item.name}}"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "{{item.changed | ternary('latest','present') }}"
+  with_items: "{{ calico_node_manifests.results }}"

roles/kubernetes-apps/network_plugin/meta/main.yml

 ---
 dependencies:
+  - role: kubernetes-apps/network_plugin/calico
+    when: kube_network_plugin == 'calico'
+    tags: calico
   - role: kubernetes-apps/network_plugin/canal
     when: kube_network_plugin == 'canal'
     tags: canal

roles/kubernetes/node/templates/kubelet.docker.service.j2

 [Unit]
 Description=Kubernetes Kubelet Server
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=docker.service docker.socket calico-node.service
-Wants=docker.socket calico-node.service
-{% else %}
 After=docker.service
 Wants=docker.socket
-{% endif %}
 
 [Service]
 EnvironmentFile={{kube_config_dir}}/kubelet.env

roles/kubernetes/node/templates/kubelet.host.service.j2

 [Unit]
 Description=Kubernetes Kubelet Server
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=docker.service docker.socket calico-node.service
-Wants=docker.socket calico-node.service
-{% else %}
 After=docker.service
 Wants=docker.socket
-{% endif %}
 
 [Service]
 EnvironmentFile={{kube_config_dir}}/kubelet.env

roles/kubernetes/node/templates/kubelet.rkt.service.j2

 [Unit]
 Description=Kubernetes Kubelet Server
 Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=calico-node.service
-Wants=network.target calico-node.service
-{% else %}
 Wants=network.target
-{% endif %}
 
 [Service]
 Restart=on-failure

roles/network_plugin/calico/defaults/main.yml

@@ -34,3 +34,8 @@ calicoctl_cpu_requests: 50m
 # Should calico ignore kernel's RPF check setting,
 # see https://github.com/projectcalico/felix/blob/ab8799eaea66627e5db7717e62fca61fd9c08646/python/calico/felix/config.py#L198
 calico_node_ignorelooserpf: false
+
+rbac_resources:
+  - sa
+  - clusterrole
+  - clusterrolebinding

roles/network_plugin/calico/tasks/main.yml

 ---
-- name: Calico | Check calicoctl version
-  run_once: true
-  set_fact:
-    legacy_calicoctl: "{{ calicoctl_image_tag | version_compare('v1.0.0', '<') }}"
-  tags: facts
+- name: Calico | Disable calico-node service if it exists
+  service:
+    name: calico-node
+    state: stopped
+    enabled: yes
+  failed_when: false
 
 - name: Calico | Write Calico cni config
   template:
@@ -38,7 +39,6 @@
     owner: root
     group: root
   changed_when: false
-  notify: restart calico-node
 
 - name: Calico | Copy cni plugins from hyperkube
   command: "{{ docker_bin_dir }}/docker run --rm -v /opt/cni/bin:/cnibindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/rsync -ac /opt/cni/bin/ /cnibindir/"
@@ -103,38 +103,7 @@
   environment:
     NO_DEFAULT_POOLS: true
   run_once: true
-  when: not legacy_calicoctl and
-         ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
-
-- name: Calico (old) | Define ipip pool argument
-  run_once: true
-  set_fact:
-    ipip_arg: "--ipip"
-  when: (legacy_calicoctl and ipip )
-  tags: facts
-
-- name: Calico (old) | Define nat-outgoing pool argument
-  run_once: true
-  set_fact:
-    nat_arg: "--nat-outgoing"
-  when: (legacy_calicoctl and
-         nat_outgoing|default(false) and not peer_with_router|default(false))
-  tags: facts
-
-- name: Calico (old) | Define calico pool task name
-  run_once: true
-  set_fact:
-    pool_task_name: "with options {{ ipip_arg|default('') }} {{ nat_arg|default('') }}"
-  when: (legacy_calicoctl and ipip_arg|default(false) or nat_arg|default(false))
-  tags: facts
-
-- name: Calico (old) | Configure calico network pool {{ pool_task_name|default('') }}
-  command: "{{ bin_dir}}/calicoctl pool add {{ kube_pods_subnet }} {{ ipip_arg|default('') }} {{ nat_arg|default('') }}"
-  environment:
-    NO_DEFAULT_POOLS: true
-  run_once: true
-  when: legacy_calicoctl and
-         ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
+  when: ("Key not found" in calico_conf.stdout or "nodes" not in calico_conf.stdout)
 
 - name: Calico | Get calico configuration from etcd
   command: |-
@@ -162,52 +131,11 @@
 - name: Calico | Set global as_num
   command: "{{ bin_dir}}/calicoctl config set asNumber {{ global_as_num }}"
   run_once: true
-  when: not legacy_calicoctl
-
-- name: Calico (old) | Set global as_num
-  command: "{{ bin_dir}}/calicoctl bgp default-node-as {{ global_as_num }}"
-  run_once: true
-  when: legacy_calicoctl
-
-- name: Calico (old) | Write calico-node systemd init file
-  template:
-    src: calico-node.service.legacy.j2
-    dest: /etc/systemd/system/calico-node.service
-  when: legacy_calicoctl
-  notify: restart calico-node
-
-- name: Calico | Write calico.env for systemd init file
-  template:
-    src: calico.env.j2
-    dest: /etc/calico/calico.env
-  when: not legacy_calicoctl
-  notify: restart calico-node
-
-- name: Calico | Write calico-node systemd init file
-  template:
-    src: calico-node.service.j2
-    dest: /etc/systemd/system/calico-node.service
-  when: not legacy_calicoctl
-  notify: restart calico-node
-
-- name: Calico | Restart calico-node if secrets changed
-  command: /bin/true
-  when: secret_changed|default(false) or etcd_secret_changed|default(false)
-  notify: restart calico-node
-
-- meta: flush_handlers
-
-- name: Calico | Enable calico-node
-  service:
-    name: calico-node
-    state: started
-    enabled: yes
 
 - name: Calico | Disable node mesh
   shell: "{{ bin_dir }}/calicoctl config set nodeToNodeMesh off"
   when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
-          and inventory_hostname in groups['k8s-cluster']
-          and not legacy_calicoctl)
+          and inventory_hostname in groups['k8s-cluster'])
   run_once: true
 
 - name: Calico | Configure peering with router(s)
@@ -220,8 +148,7 @@
    }'
    | {{ bin_dir }}/calicoctl create --skip-exists -f -
   with_items: "{{ peers|default([]) }}"
-  when: (not legacy_calicoctl and
-         peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster'])
+  when: peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster']
 
 - name: Calico | Configure peering with route reflectors
   shell: >
@@ -235,26 +162,20 @@
    }'
    | {{ bin_dir }}/calicoctl create --skip-exists -f -
   with_items: "{{ groups['calico-rr'] | default([]) }}"
-  when: (not legacy_calicoctl and
-         peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
+  when: (peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
          and hostvars[item]['cluster_id'] == cluster_id)
 
-- name: Calico (old) | Disable node mesh
-  shell: "{{ bin_dir }}/calicoctl bgp node-mesh off"
-  when: ((peer_with_router|default(false) or peer_with_calico_rr|default(false))
-          and inventory_hostname in groups['k8s-cluster']
-          and legacy_calicoctl)
-  run_once: true
-
-- name: Calico (old) | Configure peering with router(s)
-  shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ item.router_id }} as {{ item.as }}"
-  with_items: "{{ peers|default([]) }}"
-  when: (legacy_calicoctl and
-         peer_with_router|default(false) and inventory_hostname in groups['k8s-cluster'])
-
-- name: Calico (old) | Configure peering with route reflectors
-  shell: "{{ bin_dir }}/calicoctl node bgp peer add {{ hostvars[item]['calico_rr_ip']|default(hostvars[item]['ip'])|default(hostvars[item]['ansible_default_ipv4.address']) }} as {{ local_as | default(global_as_num) }}"
-  with_items: "{{ groups['calico-rr'] | default([]) }}"
-  when: (legacy_calicoctl and
-         peer_with_calico_rr|default(false) and inventory_hostname in groups['k8s-cluster']
-         and hostvars[item]['cluster_id'] == cluster_id)
+- name: Calico | Create calico manifests
+  template:
+    src: "{{item.file}}.j2"
+    dest: "{{kube_config_dir}}/{{item.file}}"
+  with_items:
+    - {name: calico-config, file: calico-config.yml, type: cm}
+    - {name: calico-node, file: calico-node.yml, type: ds}
+    - {name: calico, file: calico-node-sa.yml, type: sa}
+    - {name: calico, file: calico-cr.yml, type: clusterrole}
+    - {name: calico, file: calico-crb.yml, type: clusterrolebinding}
+  register: calico_node_manifests
+  when:
+    - inventory_hostname in groups['kube-master']
+    - rbac_enabled or item.type not in rbac_resources

roles/network_plugin/calico/templates/calico-config.yml.j2

+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: calico-config
+  namespace: {{ system_namespace }}
+data:
+  etcd_endpoints: "{{ etcd_access_endpoint }}"
+  etcd_ca: "/calico-secrets/ca_cert.crt"
+  etcd_cert: "/calico-secrets/cert.crt"
+  etcd_key: "/calico-secrets/key.pem"
+{% if calico_network_backend is defined and calico_network_backend == 'none' %}
+  cluster_type: "kubespray"
+{%- else %}
+  cluster_type: "kubespray,bgp"
+{% endif %}
+  calico_backend: "bird"
+  {%- if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false) %}
+  as: "{{ local_as }}"
+  {% endif -%}

roles/network_plugin/calico/templates/calico-cr.yml.j2

+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: calico-node
+  namespace: {{ system_namespace }}
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - nodes
+    verbs:
+      - get

roles/network_plugin/calico/templates/calico-crb.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-node
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: {{ system_namespace }}

roles/network_plugin/calico/templates/calico-node-sa.yml.j2

+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: {{ system_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"

roles/network_plugin/calico/templates/calico-node.service.j2

-[Unit]
-Description=calico-node
-After=docker.service
-Requires=docker.service
-
-[Service]
-EnvironmentFile=/etc/calico/calico.env
-ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-node
-ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
- --name=calico-node \
- -e HOSTNAME=${CALICO_HOSTNAME} \
- -e IP=${CALICO_IP} \
- -e IP6=${CALICO_IP6} \
- -e CLUSTER_TYPE=${CLUSTER_TYPE} \
- -e CALICO_NETWORKING_BACKEND=${CALICO_NETWORKING_BACKEND} \
- -e FELIX_DEFAULTENDPOINTTOHOSTACTION={{ calico_endpoint_to_host_action|default('RETURN') }} \
- -e AS=${CALICO_AS} \
- -e NO_DEFAULT_POOLS=${CALICO_NO_DEFAULT_POOLS} \
- -e CALICO_LIBNETWORK_ENABLED=${CALICO_LIBNETWORK_ENABLED} \
- -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
- -e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
- -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \
- -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
-{% if calico_node_ignorelooserpf %}
- -e FELIX_IGNORELOOSERPF=true \
-{% endif %}
- -v /var/log/calico:/var/log/calico \
- -v /run/docker/plugins:/run/docker/plugins \
- -v /lib/modules:/lib/modules \
- -v /var/run/calico:/var/run/calico \
- -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
- --memory={{ calico_node_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_node_cpu_limit|regex_replace('m', '') }} \
- {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
-
-Restart=always
-RestartSec=10s
-
-ExecStop=-{{ docker_bin_dir }}/docker stop calico-node
-
-[Install]
-WantedBy=multi-user.target

roles/network_plugin/calico/templates/calico-node.service.legacy.j2

-[Unit]
-Description=Calico per-node agent
-Documentation=https://github.com/projectcalico/calico-docker
-After=docker.service docker.socket
-Wants=docker.socket
-
-[Service]
-User=root
-PermissionsStartOnly=true
-{% if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false)%}
-ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --as={{ local_as }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }}
-{% else %}
-ExecStart={{ bin_dir }}/calicoctl node --ip={{ip | default(ansible_default_ipv4.address) }} --detach=false --node-image={{ calico_node_image_repo }}:{{ calico_node_image_tag }}
-{% endif %}
-Restart=always
-RestartSec=10s
-
-[Install]
-WantedBy=multi-user.target

roles/network_plugin/calico/templates/calico-node.yml.j2

+---
+# This manifest installs the calico/node container, as well
+# as the Calico CNI plugins and network config on
+# each master and worker node in a Kubernetes cluster.
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: calico-node
+  namespace: {{ system_namespace }}
+  labels:
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    spec:
+      hostNetwork: true
+{% if rbac_enabled %}
+      serviceAccountName: calico-node
+{% endif %}
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      containers:
+        # Runs calico/node container on each Kubernetes node.  This
+        # container programs network policy and routes on each
+        # host.
+        - name: calico-node
+          image: {{ calico_node_image_repo }}:{{ calico_node_image_tag }}
+          env:
+            # The location of the Calico etcd cluster.
+            - name: ETCD_ENDPOINTS
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_endpoints
+            # Choose the backend to use.
+            - name: CALICO_NETWORKING_BACKEND
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: calico_backend
+            # Cluster type to identify the deployment type
+            - name: CLUSTER_TYPE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cluster_type
+            # Disable file logging so `kubectl logs` works.
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "{{ calico_endpoint_to_host_action|default('RETURN') }}"
+# should be set in etcd before deployment
+#            # Configure the IP Pool from which Pod IPs will be chosen.
+#            - name: CALICO_IPV4POOL_CIDR
+#              value: "192.168.0.0/16"
+#            - name: CALICO_IPV4POOL_IPIP
+#              value: "always"
+            # Disable IPv6 on Kubernetes.
+            - name: FELIX_IPV6SUPPORT
+              value: "false"
+            # Set Felix logging to "info"
+            - name: FELIX_LOGSEVERITYSCREEN
+              value: "info"
+            # Disable autocreation of pools
+            - name: CALICO_NO_DEFAULT_POOLS
+              value: "true"
+            # Enable libnetwork
+            - name: CALICO_LIBNETWORK_ENABLED
+              value: "true"
+            # Set MTU for tunnel device used if ipip is enabled
+{% if calico_mtu is defined %}
+            - name: FELIX_IPINIPMTU
+              value: "{{ calico_mtu }}"
+{% endif %}
+            # Location of the CA certificate for etcd.
+            - name: ETCD_CA_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_ca
+            # Location of the client key for etcd.
+            - name: ETCD_KEY_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_key
+            # Location of the client certificate for etcd.
+            - name: ETCD_CERT_FILE
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: etcd_cert
+            - name: IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+          securityContext:
+            privileged: true
+          resources:
+            limits:
+              cpu: {{ calico_node_cpu_limit }}
+              memory: {{ calico_node_memory_limit }}
+            requests:
+              cpu: {{ calico_node_cpu_requests }}
+              memory: {{ calico_node_memory_requests }}
+          livenessProbe:
+            httpGet:
+              path: /liveness
+              port: 9099
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            httpGet:
+              path: /readiness
+              port: 9099
+            periodSeconds: 10
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false
+            - mountPath: /calico-secrets
+              name: etcd-certs
+      volumes:
+        # Used by calico/node.
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+        # Used to install CNI.
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        # Mount in the etcd TLS secrets.
+        - name: etcd-certs
+          hostPath:
+            path: "{{ calico_cert_dir }}"
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
+

roles/network_plugin/calico/templates/calico.env.j2

-ETCD_ENDPOINTS="{{ etcd_access_endpoint }}"
-ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt"
-ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt"
-ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem"
-CALICO_IP="{{ip | default(ansible_default_ipv4.address) }}"
-CALICO_IP6=""
-{% if calico_network_backend is defined and calico_network_backend == 'none' %}
-CLUSTER_TYPE="kubespray"
-{% else %}
-CLUSTER_TYPE="kubespray,bgp"
-{% endif %}
-{% if calico_network_backend is defined %}
-CALICO_NETWORKING_BACKEND="{{calico_network_backend }}"
-{% endif %}
-{% if inventory_hostname in groups['k8s-cluster'] and peer_with_router|default(false)%}
-CALICO_AS="{{ local_as }}"
-{% endif %}
-CALICO_NO_DEFAULT_POOLS="true"
-CALICO_LIBNETWORK_ENABLED="true"
-CALICO_HOSTNAME="{{ ansible_hostname }}"

roles/network_plugin/calico/templates/cni-calico.conf.j2

 {
   "name": "calico-k8s-network",
-{% if not legacy_calicoctl %}
   "hostname": "{{ ansible_hostname }}",
-{% endif %}
   "type": "calico",
   "etcd_endpoints": "{{ etcd_access_endpoint }}",
   "etcd_cert_file": "{{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem",