Upgrade ansible-core to 2.16.4 (#10984)

* upgrade ansible version

Needed for with_first_found to work correctly:
https://github.com/ansible/ansible/issues/70772 fixed in 2.16

* Remove unused google cloud cloud_playbook

* Fix dpkg_selection on non-existing packages

Needed since ansible-core>2.16, see:
https://github.com/ansible/ansible/commit/f10d11bcdc54c9b7edc0111eb38c59a88e396d0a

docs/ansible.md

@@ -32,7 +32,7 @@ Based on the table below and the available python version for your ansible host
 
 | Ansible Version | Python Version |
 |-----------------|----------------|
-| >= 2.15.5       | 3.9-3.11       |
+| >= 2.16.4       | 3.9-3.11       |
 
 ## Inventory
 

meta/runtime.yml

 ---
-requires_ansible: '>=2.15.5'
+requires_ansible: '>=2.16.4'

playbooks/ansible_version.yml

@@ -5,7 +5,7 @@
   become: no
   run_once: true
   vars:
-    minimal_ansible_version: 2.15.5  # 2.15 versions before 2.15.5 are known to be buggy for kubespray
+    minimal_ansible_version: 2.16.4
     maximal_ansible_version: 2.17.0
   tags: always
   tasks:

requirements.txt

-ansible==8.5.0
+ansible==9.3.0
 cryptography==41.0.4
 jinja2==3.1.2
 jmespath==1.0.1
 MarkupSafe==2.1.3
 netaddr==0.9.0
 pbr==5.11.1
-ruamel.yaml==0.17.35
+ruamel.yaml==0.18.5
 ruamel.yaml.clib==0.2.8

roles/container-engine/docker/tasks/main.yml

@@ -96,7 +96,11 @@
     name: "{{ item }}"
     selection: install
   when: ansible_pkg_mgr == 'apt'
+  register: ret
   changed_when: false
+  failed_when:
+    - ret is failed
+    - ret.msg != ( "Failed to find package '" + item + "' to perform selection 'install'." )
   with_items:
     - "{{ containerd_package }}"
     - docker-ce

tests/Makefile

@@ -22,28 +22,6 @@ create-tf:
 delete-tf:
 	./scripts/delete-tf.sh
 
-create-gce: init-gce
-	ansible-playbook cloud_playbooks/create-gce.yml -i local_inventory/hosts.cfg -c local \
-	$(ANSIBLE_LOG_LEVEL) \
-	-e @"files/${CI_JOB_NAME}.yml" \
-	-e gce_credentials_file=$(HOME)/.ssh/gce.json \
-	-e gce_project_id=$(GCE_PROJECT_ID) \
-	-e gce_service_account_email=$(GCE_ACCOUNT) \
-	-e inventory_path=$(INVENTORY) \
-	-e test_id=$(TEST_ID) \
-	-e preemptible=$(GCE_PREEMPTIBLE)
-
-
-delete-gce:
-	ansible-playbook -i $(INVENTORY) cloud_playbooks/delete-gce.yml -c local \
-	$(ANSIBLE_LOG_LEVEL) \
-	-e @"files/${CI_JOB_NAME}.yml" \
-	-e test_id=$(TEST_ID) \
-	-e gce_project_id=$(GCE_PROJECT_ID) \
-	-e gce_service_account_email=$(GCE_ACCOUNT) \
-	-e gce_credentials_file=$(HOME)/.ssh/gce.json \
-	-e inventory_path=$(INVENTORY)
-
 create-do: init-do
 	ansible-playbook cloud_playbooks/create-do.yml -i local_inventory/hosts.cfg -c local \
 	${ANSIBLE_LOG_LEVEL} \

tests/cloud_playbooks/create-gce.yml

----
-- name: Provision Google Cloud VMs
-  hosts: localhost
-  become: false
-  gather_facts: no
-  vars:
-    cloud_machine_type: g1-small
-    mode: default
-    preemptible: no
-    ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}"
-    delete_group_vars: no
-  tasks:
-    - name: Include vars for test {{ ci_job_name }}
-      include_vars: "../files/{{ ci_job_name }}.yml"
-
-    - name: Replace_test_id
-      set_fact:
-        test_name: "{{ test_id | regex_replace('\\.', '-') }}"
-
-    - name: Set instance names
-      set_fact:
-        # noqa: jinja[spacing]
-        instance_names: >-
-          {%- if mode in ['separate', 'separate-scale', 'ha', 'ha-scale'] -%}
-          k8s-{{ test_name }}-1,k8s-{{ test_name }}-2,k8s-{{ test_name }}-3
-          {%- elif mode == 'all-in-one' -%}
-          k8s-{{ test_name }}-1
-          {%- else -%}
-          k8s-{{ test_name }}-1,k8s-{{ test_name }}-2
-          {%- endif -%}
-
-    - name: Create gce instances
-      google.cloud.gcp_compute_instance:  # noqa args[module] - Probably doesn't work
-        instance_names: "{{ instance_names }}"
-        machine_type: "{{ cloud_machine_type }}"
-        image: "{{ cloud_image | default(omit) }}"
-        image_family: "{{ cloud_image_family | default(omit) }}"
-        preemptible: "{{ preemptible }}"
-        service_account_email: "{{ gce_service_account_email }}"
-        pem_file: "{{ gce_pem_file | default(omit) }}"
-        credentials_file: "{{ gce_credentials_file | default(omit) }}"
-        project_id: "{{ gce_project_id }}"
-        zone: "{{ cloud_region }}"
-        metadata: '{"test_id": "{{ test_id }}", "network": "{{ kube_network_plugin }}", "startup-script": "{{ startup_script | default("") }}"}'
-        tags: "build-{{ test_name }},{{ kube_network_plugin }}"
-        ip_forward: yes
-        service_account_permissions: ['compute-rw']
-      register: gce
-
-    - name: Add instances to host group
-      add_host:
-        hostname: "{{ item.public_ip }}"
-        groupname: "waitfor_hosts"
-      with_items: '{{ gce.instance_data }}'
-
-    - name: Template the inventory  # noqa no-relative-paths - CI inventory templates are not in role_path
-      template:
-        src: ../templates/inventory-gce.j2
-        dest: "{{ inventory_path }}"
-        mode: 0644
-
-    - name: Make group_vars directory
-      file:
-        path: "{{ inventory_path | dirname }}/group_vars"
-        state: directory
-        mode: 0755
-      when: mode in ['scale', 'separate-scale', 'ha-scale']
-
-    - name: Template fake hosts group vars  # noqa no-relative-paths - CI templates are not in role_path
-      template:
-        src: ../templates/fake_hosts.yml.j2
-        dest: "{{ inventory_path | dirname }}/group_vars/fake_hosts.yml"
-        mode: 0644
-      when: mode in ['scale', 'separate-scale', 'ha-scale']
-
-    - name: Delete group_vars directory
-      file:
-        path: "{{ inventory_path | dirname }}/group_vars"
-        state: absent
-        recurse: yes
-      when: delete_group_vars

tests/cloud_playbooks/delete-gce.yml

----
-- name: Terminate Google Cloud VMs
-  hosts: localhost
-  become: false
-  gather_facts: no
-  vars:
-    mode: default
-
-  tasks:
-    - name: Replace_test_id
-      set_fact:
-        test_name: "{{ test_id | regex_replace('\\.', '-') }}"
-
-    - name: Set instance names
-      set_fact:
-        # noqa: jinja[spacing]
-        instance_names: >-
-          {%- if mode in ['separate', 'ha'] -%}
-          k8s-{{ test_name }}-1,k8s-{{ test_name }}-2,k8s-{{ test_name }}-3
-          {%- else -%}
-          k8s-{{ test_name }}-1,k8s-{{ test_name }}-2
-          {%- endif -%}
-
-    - name: Stop gce instances  # noqa args[module] - Probably doesn't work
-      google.cloud.gcp_compute_instance:
-        instance_names: "{{ instance_names }}"
-        image: "{{ cloud_image | default(omit) }}"
-        service_account_email: "{{ gce_service_account_email }}"
-        pem_file: "{{ gce_pem_file | default(omit) }}"
-        credentials_file: "{{ gce_credentials_file | default(omit) }}"
-        project_id: "{{ gce_project_id }}"
-        zone: "{{ cloud_region | default('europe-west1-b') }}"
-        state: 'stopped'
-      async: 120
-      poll: 3
-      register: gce
-
-    - name: Delete gce instances  # noqa args[module] - Probably doesn't work
-      google.cloud.gcp_compute_instance:
-        instance_names: "{{ instance_names }}"
-        image: "{{ cloud_image | default(omit) }}"
-        service_account_email: "{{ gce_service_account_email }}"
-        pem_file: "{{ gce_pem_file | default(omit) }}"
-        credentials_file: "{{ gce_credentials_file | default(omit) }}"
-        project_id: "{{ gce_project_id }}"
-        zone: "{{ cloud_region | default('europe-west1-b') }}"
-        state: 'absent'
-      async: 120
-      poll: 3
-      register: gce

tests/cloud_playbooks/upload-logs-gcs.yml

----
-- name: Upload logs to GCS
-  hosts: localhost
-  become: false
-  gather_facts: no
-
-  vars:
-    expire_days: 2
-
-  tasks:
-    - name: Generate uniq bucket name prefix
-      raw: date +%Y%m%d
-      changed_when: false
-      register: out
-
-    - name: Replace_test_id
-      set_fact:
-        test_name: "kargo-ci-{{ out.stdout_lines[0] }}"
-
-    - name: Set file_name for logs
-      set_fact:
-        file_name: "{{ ostype }}-{{ kube_network_plugin }}-{{ commit }}-logs.tar.gz"
-
-    - name: Create a bucket
-      community.google.gc_storage:
-        bucket: "{{ test_name }}"
-        mode: create
-        permission: public-read
-        gs_access_key: "{{ gs_key }}"
-        gs_secret_key: "{{ gs_skey }}"
-      no_log: True
-
-    - name: Create a lifecycle template for the bucket
-      template:
-        src: gcs_life.json.j2
-        dest: "{{ dir }}/gcs_life.json"
-        mode: 0644
-
-    - name: Create a boto config to access GCS
-      template:
-        src: boto.j2
-        dest: "{{ dir }}/.boto"
-        mode: 0640
-      no_log: True
-
-    - name: Download gsutil cp installer
-      get_url:
-        url: https://dl.google.com/dl/cloudsdk/channels/rapid/install_google_cloud_sdk.bash
-        dest: "{{ dir }}/gcp-installer.sh"
-        mode: 0644
-
-    - name: Get gsutil tool
-      command: "{{ dir }}/gcp-installer.sh"
-      environment:
-        CLOUDSDK_CORE_DISABLE_PROMPTS: "1"
-        CLOUDSDK_INSTALL_DIR: "{{ dir }}"
-      no_log: True
-      failed_when: false
-
-    - name: Apply the lifecycle rules
-      command: "{{ dir }}/google-cloud-sdk/bin/gsutil lifecycle set {{ dir }}/gcs_life.json gs://{{ test_name }}"
-      changed_when: false
-      environment:
-        BOTO_CONFIG: "{{ dir }}/.boto"
-      no_log: True
-
-    - name: Upload collected diagnostic info
-      community.google.gc_storage:
-        bucket: "{{ test_name }}"
-        mode: put
-        permission: public-read
-        object: "{{ file_name }}"
-        src: "{{ dir }}/logs.tar.gz"
-        headers: '{"Content-Encoding": "x-gzip"}'
-        gs_access_key: "{{ gs_key }}"
-        gs_secret_key: "{{ gs_skey }}"
-        expiration: "{{ expire_days * 36000 | int }}"
-      failed_when: false
-      no_log: True
-
-    - debug:  # noqa name[missing]
-        msg: "A public url https://storage.googleapis.com/{{ test_name }}/{{ file_name }}"

tests/requirements.txt

 -r ../requirements.txt
-ansible-lint==6.20.3
+ansible-lint==6.22.2
 apache-libcloud==3.8.0
 ara[server]==1.7.0
 dopy==0.3.7