tests: cleanup stale packet namespace automatically (#10245)

* tests: cleanup stale packet namespace automatically

Cancelled job on Gitlab can produce stale VMs as the delete playbook
will never be executed. This commits allow removing old vms by getting
all the namespace created from the same branch with an older pipeline
id.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: cleanup stale packet namespace after 2 hours

This ensure that we don't have any packet namespace remaining for more
than 2 hours. All the jobs complete usually within 30min-1hour so 2
hours is enough to detect a stale namespace.

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: ignore vm cleanup failure

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

* tests: use pipeline_id var instead of fetching namespace for cleanup packet vm

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

---------

Signed-off-by: Arthur Outhenin-Chalandre <arthur.outhenin-chalandre@proton.ch>

.gitlab-ci/packet.yml

@@ -23,6 +23,14 @@
   allow_failure: true
   extends: .packet
 
+packet_cleanup_old:
+  stage: deploy-part1
+  extends: .packet_periodic
+  script:
+    - cd tests
+    - make cleanup-packet
+  after_script: []
+
 # The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-aio:
   stage: deploy-part1

tests/Makefile

@@ -64,6 +64,8 @@ create-packet: init-packet
 	$(ANSIBLE_LOG_LEVEL) \
 	-e @"files/${CI_JOB_NAME}.yml" \
 	-e test_id=$(TEST_ID) \
+	-e branch="$(CI_COMMIT_BRANCH)" \
+	-e pipeline_id="$(CI_PIPELINE_ID)" \
 	-e inventory_path=$(INVENTORY)
 
 delete-packet:
@@ -71,8 +73,14 @@ delete-packet:
 	$(ANSIBLE_LOG_LEVEL) \
 	-e @"files/${CI_JOB_NAME}.yml" \
 	-e test_id=$(TEST_ID) \
+	-e branch="$(CI_COMMIT_BRANCH)" \
+	-e pipeline_id="$(CI_PIPELINE_ID)" \
 	-e inventory_path=$(INVENTORY)
 
+cleanup-packet:
+	ansible-playbook cloud_playbooks/cleanup-packet.yml -c local \
+	$(ANSIBLE_LOG_LEVEL)
+
 create-vagrant:
 	vagrant up
 	find / -name vagrant_ansible_inventory

tests/cloud_playbooks/cleanup-packet.yml

+---
+
+- hosts: localhost
+  gather_facts: no
+  become: true
+  roles:
+    - { role: cleanup-packet-ci }

tests/cloud_playbooks/roles/cleanup-packet-ci/tasks/main.yml

+---
+
+- name: Fetch a list of namespaces
+  kubernetes.core.k8s_info:
+    api_version: v1
+    kind: Namespace
+    label_selectors:
+      - cijobs = true
+  register: namespaces
+
+- name: Delete stale namespaces for more than 2 hours
+  command: "kubectl delete namespace {{ item.metadata.name }}"
+  failed_when: false
+  loop: "{{ namespaces.resources }}"
+  when:
+    - (now() - (item.metadata.creationTimestamp | to_datetime("%Y-%m-%dT%H:%M:%SZ"))).total_seconds() >= 7200

tests/cloud_playbooks/roles/packet-ci/tasks/cleanup-old-vms.yml

+---
+
+- name: Fetch a list of namespaces
+  kubernetes.core.k8s_info:
+    api_version: v1
+    kind: Namespace
+    label_selectors:
+      - cijobs = true
+      - branch = {{ branch }}
+  register: namespaces
+
+- name: Delete older namespaces
+  command: "kubectl delete namespace {{ item.metadata.name }}"
+  failed_when: false
+  loop: "{{ namespaces.resources }}"
+  when:
+    - (item.metadata.labels.pipeline_id | int) < (pipeline_id | int)

tests/cloud_playbooks/roles/packet-ci/tasks/create-vms.yml

 ---
 
 - name: "Create CI namespace {{ test_name }} for test vms"
-  command: "kubectl create namespace {{ test_name }}"
+  shell: |-
+    kubectl create namespace {{ test_name }} &&
+      kubectl label namespace {{ test_name }} cijobs=true branch="{{ branch }}" pipeline_id="{{ pipeline_id }}"
   changed_when: false
 
 - name: "Create temp dir /tmp/{{ test_name }} for CI files"

tests/cloud_playbooks/roles/packet-ci/tasks/main.yml

@@ -7,6 +7,8 @@
   set_fact:
     vm_count: "{%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale', 'ha-recover', 'ha-recover-noquorum'] -%}{{ 3|int }}{%- elif mode == 'aio' -%}{{ 1|int }}{%- else -%}{{ 2|int }}{%- endif -%}"
 
+- import_tasks: cleanup-old-vms.yml
+
 - import_tasks: create-vms.yml
   when:
     - not vm_cleanup