From 023108a7330c2d944dedfd2a44c0f9ca58589d77 Mon Sep 17 00:00:00 2001
From: Matthew Mosesohn <matthew.mosesohn@gmail.com>
Date: Thu, 8 Aug 2019 17:37:22 +0300
Subject: [PATCH] Refactor calico route reflector to run in k8s cluster (#4975)

* Refactor calico-rr to run in k8s cluster with taint

Change-Id: I75a3169ff5b36ce8302fc7ef1c32d3eb697b5afa

* add preinstall checks

* rework calico/rr role

Change-Id: I2f0a7e6cb77cf91ad4a615923680760d2e5d9ca8

* add empty calico-rr group

Change-Id: I006c0a60db9b72d02245bf8fdfabcf982144a5ad
---
 cluster.yml                                   |  18 +--
 docs/calico.md                                |   9 +-
 inventory/local/hosts.ini                     |   1 +
 inventory/sample/inventory.ini                |   3 +
 .../templates/kubeadm-client.conf.v1alpha3.j2 |   5 +
 .../templates/kubeadm-client.conf.v1beta1.j2  |   5 +
 .../preinstall/tasks/0020-verify-settings.yml |  20 +++
 .../calico/rr/defaults/main.yml               |  13 +-
 .../calico/rr/handlers/main.yml               |  15 --
 roles/network_plugin/calico/rr/tasks/main.yml |  95 +++----------
 roles/network_plugin/calico/rr/tasks/pre.yml  |  15 ++
 .../templates/calico-rr-containerd.service.j2 |  27 ----
 .../rr/templates/calico-rr-docker.service.j2  |  28 ----
 .../calico/rr/templates/calico-rr.env.j2      |   6 -
 roles/network_plugin/calico/tasks/install.yml | 128 ++++++++++--------
 .../roles/packet-ci/templates/inventory.j2    |   3 +
 tests/templates/inventory-aws.j2              |   3 +
 tests/templates/inventory-do.j2               |   3 +
 tests/templates/inventory-gce.j2              |   3 +
 19 files changed, 170 insertions(+), 230 deletions(-)
 delete mode 100644 roles/network_plugin/calico/rr/handlers/main.yml
 create mode 100644 roles/network_plugin/calico/rr/tasks/pre.yml
 delete mode 100644 roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2
 delete mode 100644 roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2
 delete mode 100644 roles/network_plugin/calico/rr/templates/calico-rr.env.j2

diff --git a/cluster.yml b/cluster.yml
index bc0580331..3e8b78115 100644
--- a/cluster.yml
+++ b/cluster.yml
@@ -19,14 +19,14 @@
     - { role: kubespray-defaults}
     - { role: bastion-ssh-config, tags: ["localhost", "bastion"]}
 
-- hosts: k8s-cluster:etcd:calico-rr
+- hosts: k8s-cluster:etcd
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   gather_facts: false
   roles:
     - { role: kubespray-defaults}
     - { role: bootstrap-os, tags: bootstrap-os}
 
-- hosts: k8s-cluster:etcd:calico-rr
+- hosts: k8s-cluster:etcd
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
@@ -46,7 +46,7 @@
         etcd_events_cluster_setup: "{{ etcd_events_cluster_enabled }}"
       when: not etcd_kubeadm_enabled| default(false)
 
-- hosts: k8s-cluster:calico-rr
+- hosts: k8s-cluster
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
     - { role: kubespray-defaults}
@@ -79,6 +79,12 @@
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: network_plugin, tags: network }
 
+- hosts: calico-rr
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: network_plugin/calico/rr, tags: ['network', 'calico_rr']}
+
 - hosts: kube-master[0]
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
@@ -95,12 +101,6 @@
     - { role: kubernetes-apps/ingress_controller, tags: ingress-controller }
     - { role: kubernetes-apps/external_provisioner, tags: external-provisioner }
 
-- hosts: calico-rr
-  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
-  roles:
-    - { role: kubespray-defaults}
-    - { role: network_plugin/calico/rr, tags: network }
-
 - hosts: kube-master
   any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
   roles:
diff --git a/docs/calico.md b/docs/calico.md
index 5eedef8ac..52586b18f 100644
--- a/docs/calico.md
+++ b/docs/calico.md
@@ -119,13 +119,13 @@ recommended here:
 
 You need to edit your inventory and add:
 
-* `calico-rr` group with nodes in it. At the moment it's incompatible with
-  `kube-node` due to BGP port conflict with `calico-node` container. So you
-  should not have nodes in both `calico-rr` and `kube-node` groups.
+* `calico-rr` group with nodes in it. `calico-rr` can be combined with
+  `kube-node` and/or `kube-master`. `calico-rr` group also must be a child
+   group of `k8s-cluster` group.
 * `cluster_id` by route reflector node/group (see details
 [here](https://hub.docker.com/r/calico/routereflector/))
 
-Here's an example of Kubespray inventory with route reflectors:
+Here's an example of Kubespray inventory with standalone route reflectors:
 
 ```
 [all]
@@ -154,6 +154,7 @@ node5
 [k8s-cluster:children]
 kube-node
 kube-master
+calico-rr
 
 [calico-rr]
 rr0
diff --git a/inventory/local/hosts.ini b/inventory/local/hosts.ini
index 425ad23ef..7834d27c0 100644
--- a/inventory/local/hosts.ini
+++ b/inventory/local/hosts.ini
@@ -12,3 +12,4 @@ node1
 [k8s-cluster:children]
 kube-node
 kube-master
+calico-rr
diff --git a/inventory/sample/inventory.ini b/inventory/sample/inventory.ini
index 8e32a3a75..e5a53b05a 100644
--- a/inventory/sample/inventory.ini
+++ b/inventory/sample/inventory.ini
@@ -28,6 +28,9 @@
 # node5
 # node6
 
+[calico-rr]
+
 [k8s-cluster:children]
 kube-master
 kube-node
+calico-rr
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
index 81efb98fc..0110c058c 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2
@@ -17,3 +17,8 @@ discoveryTokenUnsafeSkipCAVerification: true
 nodeRegistration:
   name: {{ kube_override_hostname }}
   criSocket: {{ cri_socket }}
+{% if 'calico-rr' in group_names and 'kube-node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/calico-rr
+{% endif %}
diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2
index 66583412b..75f27fad3 100644
--- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2
+++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2
@@ -21,3 +21,8 @@ caCertPath: {{ kube_cert_dir }}/ca.crt
 nodeRegistration:
   name: {{ kube_override_hostname }}
   criSocket: {{ cri_socket }}
+{% if 'calico-rr' in group_names and 'kube-node' not in group_names %}
+  taints:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/calico-rr
+{% endif %}
diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
index 48938d06b..c52d0fa21 100644
--- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
+++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
@@ -166,6 +166,26 @@
     - inventory_hostname == groups['kube-master'][0]
   run_once: yes
 
+- name: "Check that cluster_id is set if calico_rr enabled"
+  assert:
+    that:
+      - cluster_id is defined
+    msg: "A unique cluster_id is required if using calico_rr"
+  when:
+    - kube_network_plugin == 'calico'
+    - peer_with_calico_rr
+    - inventory_hostname == groups['kube-master'][0]
+  run_once: yes
+
+- name: "Check that calico_rr nodes are in k8s-cluster group"
+  assert:
+    that:
+      - '"k8s-cluster" in group_names'
+    msg: "calico-rr must be a child group of k8s-cluster group"
+  when:
+    - kube_network_plugin == 'calico'
+    - '"calico-rr" in group_names'
+
 - name: "Check that kube_service_addresses is a network range"
   assert:
     that:
diff --git a/roles/network_plugin/calico/rr/defaults/main.yml b/roles/network_plugin/calico/rr/defaults/main.yml
index 4871f34a8..dedda197c 100644
--- a/roles/network_plugin/calico/rr/defaults/main.yml
+++ b/roles/network_plugin/calico/rr/defaults/main.yml
@@ -2,15 +2,4 @@
 # Global as_num (/calico/bgp/v1/global/as_num)
 # should be the same as in calico role
 global_as_num: "64512"
-
-calico_cert_dir: /etc/calico/certs
-
-# Limits for apps
-calico_rr_memory_limit: 1000M
-calico_rr_cpu_limit: 300m
-calico_rr_memory_requests: 128M
-calico_rr_cpu_requests: 150m
-
-kube_etcd_cacert_file: ca.pem
-kube_etcd_cert_file: node-{{ inventory_hostname }}.pem
-kube_etcd_key_file: node-{{ inventory_hostname }}-key.pem
+calico_baremetal_nodename: "{{ kube_override_hostname | default(inventory_hostname) }}"
diff --git a/roles/network_plugin/calico/rr/handlers/main.yml b/roles/network_plugin/calico/rr/handlers/main.yml
deleted file mode 100644
index 1436649e3..000000000
--- a/roles/network_plugin/calico/rr/handlers/main.yml
+++ /dev/null
@@ -1,15 +0,0 @@
----
-- name: restart calico-rr
-  command: /bin/true
-  notify:
-    - Calico-rr | reload systemd
-    - Calico-rr | reload calico-rr
-
-- name: Calico-rr | reload systemd
-  systemd:
-    daemon_reload: true
-
-- name: Calico-rr | reload calico-rr
-  service:
-    name: calico-rr
-    state: restarted
diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml
index 2b023a2aa..0c93a5506 100644
--- a/roles/network_plugin/calico/rr/tasks/main.yml
+++ b/roles/network_plugin/calico/rr/tasks/main.yml
@@ -1,82 +1,29 @@
 ---
-# Required from inventory:
-#   calico_rr_ip - which specific IP to use for RR, defaults to
-#   "ip" from inventory or "ansible_default_ipv4.address"
+- name: Calico-rr | Pre-upgrade tasks
+  include_tasks: pre.yml
 
-- name: Calico-rr | Set IP fact
-  set_fact:
-    rr_ip: "{{ calico_rr_ip | default(ip) | default(fallback_ips[inventory_hostname]) }}"
+- name: Calico-rr | Fetch current node object
+  command: "{{ bin_dir }}/calicoctl.sh get node {{ inventory_hostname }} -oyaml"
+  register: calico_rr_node
 
-- name: Calico-rr | Create calico certs directory
-  file:
-    dest: "{{ calico_cert_dir }}"
-    state: directory
-    mode: 0750
-    owner: root
-    group: root
-
-- name: Calico-rr | Link etcd certificates for calico-node
-  file:
-    src: "{{ etcd_cert_dir }}/{{ item.s }}"
-    dest: "{{ calico_cert_dir }}/{{ item.d }}"
-    state: hard
-    force: yes
-  with_items:
-    - {s: "{{ kube_etcd_cacert_file }}", d: "ca_cert.crt"}
-    - {s: "{{ kube_etcd_cert_file }}", d: "cert.crt"}
-    - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
-
-- name: Calico-rr | Create dir for logs
-  file:
-    path: /var/log/calico-rr
-    state: directory
-    mode: 0755
-    owner: root
-    group: root
-
-- name: Calico-rr | Write calico-rr.env for systemd init file
-  template:
-    src: calico-rr.env.j2
-    dest: /etc/calico/calico-rr.env
-  notify: restart calico-rr
-
-- name: Calico-rr | Write calico-rr systemd init file
-  template:
-    src: calico-rr-docker.service.j2
-    dest: /etc/systemd/system/calico-rr.service
-  notify: restart calico-rr
-  when:
-    - container_manager in ['crio', 'docker', 'rkt']
-
-- name: Calico-rr | Write calico-rr systemd init file
-  template:
-    src: calico-rr-containerd.service.j2
-    dest: /etc/systemd/system/calico-rr.service
-  notify: restart calico-rr
-  when:
-    - container_manager == 'containerd'
+# FIXME(mattymo): Use jsonpatch when ansible/ansible#52931 is merged
+- name: Calico-rr | Set route reflector cluster ID
+  shell: >-
+    echo -e '{{ calico_rr_node.stdout }}' |
+    sed '/bgp:/a \ \ \ \ routeReflectorClusterID: {{ cluster_id }}'
+  register: calico_rr_node
+  when: '("routeReflectorClusterID: " + cluster_id|string) not in calico_rr_node.stdout_lines'
 
 - name: Calico-rr | Configure route reflector
-  command: |-
-    {{ bin_dir }}/etcdctl \
-    --endpoints={{ etcd_access_addresses }} \
-    put /calico/bgp/v1/rr_v4/{{ rr_ip }} \
-    '{
-       "ip": "{{ rr_ip }}",
-       "cluster_id": "{{ cluster_id }}"
-     }'
-  environment:
-    ETCDCTL_API: 3
-    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}.pem"
-    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ groups['etcd'][0] }}-key.pem"
+  shell: |-
+    echo -e '{{ calico_rr_node.stdout }}' |
+    {{ bin_dir }}/calicoctl.sh replace -f-
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
-  delegate_to: "{{ groups['etcd'][0] }}"
-
-- meta: flush_handlers
 
-- name: Calico-rr | Enable calico-rr
-  service:
-    name: calico-rr
-    state: started
-    enabled: yes
+- name: Calico-rr | Set label for route reflector
+  command: >-
+    {{ bin_dir }}/calicoctl.sh label node {{ inventory_hostname }}
+    'i-am-a-route-reflector=true' --overwrite
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
diff --git a/roles/network_plugin/calico/rr/tasks/pre.yml b/roles/network_plugin/calico/rr/tasks/pre.yml
new file mode 100644
index 000000000..d8dbd8072
--- /dev/null
+++ b/roles/network_plugin/calico/rr/tasks/pre.yml
@@ -0,0 +1,15 @@
+---
+- name: Calico-rr | Disable calico-rr service if it exists
+  service:
+    name: calico-rr
+    state: stopped
+    enabled: no
+  failed_when: false
+
+- name: Calico-rr | Delete obsolete files
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - /etc/calico/calico-rr.env
+    - /etc/systemd/system/calico-rr.service
diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2
deleted file mode 100644
index db719afd3..000000000
--- a/roles/network_plugin/calico/rr/templates/calico-rr-containerd.service.j2
+++ /dev/null
@@ -1,27 +0,0 @@
-[Unit]
-Description=calico-rr
-After=containerd.service
-Requires=containerd.service
-
-[Service]
-EnvironmentFile=/etc/calico/calico-rr.env
-ExecStartPre=-{{ containerd_bin_dir }}/ctr t delete -f calico-rr
-ExecStart={{ containerd_bin_dir }}/ctr run --net-host --privileged \
- --env IP=${IP} \
- --env IP6=${IP6} \
- --env ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
- --env ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
- --env ETCD_CERT_FILE=${ETCD_CERT_FILE} \
- --env ETCD_KEY_FILE=${ETCD_KEY_FILE} \
- --mount type=bind,src=/var/log/calico-rr,dst=/var/log/calico,options=rbind:rw \
- --mount type=bind,src={{ calico_cert_dir }},dst={{ calico_cert_dir }},options=rbind:ro \
- {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} \
- calico-rr
-
-Restart=always
-RestartSec=10s
-
-ExecStop=-{{ containerd_bin_dir }}/ctr c rm calico-rr
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2
deleted file mode 100644
index f6da04a4d..000000000
--- a/roles/network_plugin/calico/rr/templates/calico-rr-docker.service.j2
+++ /dev/null
@@ -1,28 +0,0 @@
-[Unit]
-Description=calico-rr
-After=docker.service
-Requires=docker.service
-
-[Service]
-EnvironmentFile=/etc/calico/calico-rr.env
-ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr
-ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \
- --name=calico-rr \
- -e IP=${IP} \
- -e IP6=${IP6} \
- -e ETCD_ENDPOINTS=${ETCD_ENDPOINTS} \
- -e ETCD_CA_CERT_FILE=${ETCD_CA_CERT_FILE} \
- -e ETCD_CERT_FILE=${ETCD_CERT_FILE} \
- -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \
- -v /var/log/calico-rr:/var/log/calico \
- -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \
- --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \
- {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }}
-
-Restart=always
-RestartSec=10s
-
-ExecStop=-{{ docker_bin_dir }}/docker stop calico-rr
-
-[Install]
-WantedBy=multi-user.target
diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.env.j2
deleted file mode 100644
index 1cdb2659c..000000000
--- a/roles/network_plugin/calico/rr/templates/calico-rr.env.j2
+++ /dev/null
@@ -1,6 +0,0 @@
-ETCD_ENDPOINTS="{{ etcd_access_addresses }}"
-ETCD_CA_CERT_FILE="{{ calico_cert_dir }}/ca_cert.crt"
-ETCD_CERT_FILE="{{ calico_cert_dir }}/cert.crt"
-ETCD_KEY_FILE="{{ calico_cert_dir }}/key.pem"
-IP="{{ rr_ip }}"
-IP6=""
diff --git a/roles/network_plugin/calico/tasks/install.yml b/roles/network_plugin/calico/tasks/install.yml
index 7ee560095..c4b03822b 100644
--- a/roles/network_plugin/calico/tasks/install.yml
+++ b/roles/network_plugin/calico/tasks/install.yml
@@ -163,16 +163,16 @@
 
 - name: Calico | Configure peering with router(s) at global scope
   shell: >
-   echo '{
-   "apiVersion": "projectcalico.org/v3",
-   "kind": "BGPPeer",
-   "metadata": {
-      "name": "global-{{ item.router_id }}"
-   },
-   "spec": {
-      "asNumber": "{{ item.as }}",
-      "peerIP": "{{ item.router_id }}"
-   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
+    echo '{
+    "apiVersion": "projectcalico.org/v3",
+    "kind": "BGPPeer",
+    "metadata": {
+       "name": "global-{{ item.router_id }}"
+    },
+    "spec": {
+       "asNumber": "{{ item.as }}",
+       "peerIP": "{{ item.router_id }}"
+    }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
@@ -181,6 +181,46 @@
     - inventory_hostname == groups['kube-master'][0]
     - peer_with_router|default(false)
 
+- name: Calico | Configure peering with route reflectors at global scope
+  shell: |
+    echo '{
+    "apiVersion": "projectcalico.org/v3",
+    "kind": "BGPPeer",
+    "metadata": {
+       "name": "peer-to-rrs"
+    },
+    "spec": {
+       "nodeSelector": "!has(i-am-a-route-reflector)",
+       "peerSelector": "has(i-am-a-route-reflector)"
+    }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico-rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - peer_with_calico_rr|default(false)
+
+- name: Calico | Configure route reflectors to peer with each other
+  shell: >
+    echo '{
+    "apiVersion": "projectcalico.org/v3",
+    "kind": "BGPPeer",
+    "metadata": {
+       "name": "rr-mesh"
+    },
+    "spec": {
+       "nodeSelector": "has(i-am-a-route-reflector)",
+       "peerSelector": "has(i-am-a-route-reflector)"
+    }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  with_items:
+    - "{{ groups['calico-rr'] | default([]) }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - peer_with_calico_rr|default(false)
+
 - name: Calico | Create calico manifests
   template:
     src: "{{ item.file }}.j2"
@@ -234,18 +274,18 @@
 
 - name: Calico | Configure node asNumber for per node peering
   shell: >
-   echo '{
-   "apiVersion": "projectcalico.org/v3",
-   "kind": "Node",
-   "metadata": {
-      "name": "{{ inventory_hostname }}"
-   },
-   "spec": {
-      "bgp": {
-        "asNumber": "{{ local_as }}"
-      },
-      "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
-   }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }}
+    echo '{
+    "apiVersion": "projectcalico.org/v3",
+    "kind": "Node",
+    "metadata": {
+       "name": "{{ inventory_hostname }}"
+    },
+    "spec": {
+       "bgp": {
+         "asNumber": "{{ local_as }}"
+       },
+       "orchRefs":[{"nodeName":"{{ inventory_hostname }}","orchestrator":"k8s"}]
+    }}' | {{ bin_dir }}/calicoctl.sh {{ 'apply -f -' if calico_datastore == "kdd" else 'create --skip-exists -f -' }}
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   when:
@@ -256,17 +296,17 @@
 
 - name: Calico | Configure peering with router(s) at node scope
   shell: >
-   echo '{
-   "apiVersion": "projectcalico.org/v3",
-   "kind": "BGPPeer",
-   "metadata": {
-      "name": "{{ inventory_hostname }}-{{ item.router_id }}"
-   },
-   "spec": {
-      "asNumber": "{{ item.as }}",
-      "node": "{{ inventory_hostname }}",
-      "peerIP": "{{ item.router_id }}"
-   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
+    echo '{
+    "apiVersion": "projectcalico.org/v3",
+    "kind": "BGPPeer",
+    "metadata": {
+       "name": "{{ inventory_hostname }}-{{ item.router_id }}"
+    },
+    "spec": {
+       "asNumber": "{{ item.as }}",
+       "node": "{{ inventory_hostname }}",
+       "peerIP": "{{ item.router_id }}"
+    }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
   retries: 4
   delay: "{{ retry_stagger | random + 3 }}"
   with_items:
@@ -274,25 +314,3 @@
   when:
     - peer_with_router|default(false)
     - inventory_hostname in groups['k8s-cluster']
-
-- name: Calico | Configure peering with route reflectors
-  shell: >
-   echo '{
-   "apiVersion": "projectcalico.org/v3",
-   "kind": "BGPPeer",
-   "metadata": {
-      "name": "{{ inventory_hostname }}-{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
-   },
-   "spec": {
-      "asNumber": "{{ local_as | default(global_as_num) }}",
-      "node": "{{ inventory_hostname }}",
-      "peerIP": "{{ hostvars[item]["calico_rr_ip"]|default(hostvars[item]["ip"])|default(fallback_ips[item]) }}"
-   }}' | {{ bin_dir }}/calicoctl.sh create --skip-exists -f -
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  with_items:
-    - "{{ groups['calico-rr'] | default([]) }}"
-  when:
-    - peer_with_calico_rr|default(false)
-    - inventory_hostname in groups['k8s-cluster']
-    - hostvars[item]['cluster_id'] == cluster_id
diff --git a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2 b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
index a2f301def..82293e0cd 100644
--- a/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
+++ b/tests/cloud_playbooks/roles/packet-ci/templates/inventory.j2
@@ -50,5 +50,8 @@ instance-1
 [k8s-cluster:children]
 kube-node
 kube-master
+calico-rr
+
+[calico-rr]
 
 [fake_hosts]
diff --git a/tests/templates/inventory-aws.j2 b/tests/templates/inventory-aws.j2
index ee89bb5a4..92f107f65 100644
--- a/tests/templates/inventory-aws.j2
+++ b/tests/templates/inventory-aws.j2
@@ -22,3 +22,6 @@ node2
 [k8s-cluster:children]
 kube-node
 kube-master
+calico-rr
+
+[calico-rr]
diff --git a/tests/templates/inventory-do.j2 b/tests/templates/inventory-do.j2
index 95b6f3027..83a749afc 100644
--- a/tests/templates/inventory-do.j2
+++ b/tests/templates/inventory-do.j2
@@ -43,6 +43,9 @@
 {{droplets.results[0].droplet.name}}
 {% endif %}
 
+[calico-rr]
+
 [k8s-cluster:children]
 kube-node
 kube-master
+calico-rr
diff --git a/tests/templates/inventory-gce.j2 b/tests/templates/inventory-gce.j2
index 896cc157d..503bb4091 100644
--- a/tests/templates/inventory-gce.j2
+++ b/tests/templates/inventory-gce.j2
@@ -66,6 +66,9 @@
 [k8s-cluster:children]
 kube-node
 kube-master
+calico-rr
+
+[calico-rr]
 
 {% if mode is defined and mode in ["scale", "separate-scale", "ha-scale"] %}
 [fake_hosts]
-- 
GitLab