From c6e5314fab3ee2e05590b69f578a4fb1ae1903e5 Mon Sep 17 00:00:00 2001
From: Cristian Calin <6627509+cristicalin@users.noreply.github.com>
Date: Mon, 14 Feb 2022 23:19:32 +0200
Subject: [PATCH] implement download mirrors support (#8474)
* [download] add mechanism to support mirrors
* [calico] support alternate download url
---
roles/download/defaults/main.yml | 4 ++++
roles/download/tasks/download_file.yml | 31 +++++++++++++++++++++++++-
2 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
index ca0048673..dc06f73d9 100644
--- a/roles/download/defaults/main.yml
+++ b/roles/download/defaults/main.yml
@@ -138,6 +138,7 @@ etcd_download_url: "https://github.com/etcd-io/etcd/releases/download/{{ etcd_ve
flannel_cni_download_url: "https://github.com/flannel-io/cni-plugin/releases/download/{{ flannel_cni_version }}/flannel-{{ image_arch }}"
cni_download_url: "https://github.com/containernetworking/plugins/releases/download/{{ cni_version }}/cni-plugins-linux-{{ image_arch }}-{{ cni_version }}.tgz"
calicoctl_download_url: "https://github.com/projectcalico/calicoctl/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
+calicoctl_alternate_download_url: "https://github.com/projectcalico/calico/releases/download/{{ calico_ctl_version }}/calicoctl-linux-{{ image_arch }}"
calico_crds_download_url: "https://github.com/projectcalico/calico/archive/{{ calico_version }}.tar.gz"
crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/{{ crictl_version }}/crictl-{{ crictl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz"
helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz"
@@ -1120,6 +1121,9 @@ downloads:
dest: "{{ local_release_dir }}/calicoctl"
sha256: "{{ calicoctl_binary_checksum }}"
url: "{{ calicoctl_download_url }}"
+ mirrors:
+ - "{{ calicoctl_alternate_download_url }}"
+ - "{{ calicoctl_download_url }}"
unarchive: false
owner: "root"
mode: "0755"
diff --git a/roles/download/tasks/download_file.yml b/roles/download/tasks/download_file.yml
index f7dcfda10..d84098648 100644
--- a/roles/download/tasks/download_file.yml
+++ b/roles/download/tasks/download_file.yml
@@ -47,11 +47,40 @@
- download_force_cache
- not download_localhost
+ # We check a number of mirrors that may hold the file and pick a working one at random
+ # This task will avoid logging it's parameters to not leak environment passwords in the log
+ - name: download_file | Validate mirrors
+ uri:
+ url: "{{ mirror }}"
+ method: HEAD
+ validate_certs: "{{ download_validate_certs }}"
+ url_username: "{{ download.username | default(omit) }}"
+ url_password: "{{ download.password | default(omit) }}"
+ force_basic_auth: "{{ download.force_basic_auth | default(omit) }}"
+ delegate_to: "{{ download_delegate if download_force_cache else inventory_hostname }}"
+ run_once: "{{ download_force_cache }}"
+ register: uri_result
+ until: uri_result is success
+ retries: 4
+ delay: "{{ retry_stagger | default(5) }}"
+ environment: "{{ proxy_env }}"
+ no_log: true
+ loop: "{{ download.mirrors | default([download.url]) }}"
+ loop_control:
+ loop_var: mirror
+ ignore_errors: true
+
+ # Ansible 2.9 requires we convert a generator to a list
+ - name: download_file | Get the list of working mirrors
+ set_fact:
+ valid_mirror_urls: "{{ uri_result.results | selectattr('failed', 'eq', False) | map(attribute='mirror') | list }}"
+ delegate_to: "{{ download_delegate if download_force_cache else inventory_hostname }}"
+
# This must always be called, to check if the checksum matches. On no-match the file is re-downloaded.
# This task will avoid logging it's parameters to not leak environment passwords in the log
- name: download_file | Download item
get_url:
- url: "{{ download.url }}"
+ url: "{{ valid_mirror_urls | random }}"
dest: "{{ file_path_cached if download_force_cache else download.dest }}"
owner: "{{ omit if download_localhost else (download.owner | default(omit)) }}"
mode: "{{ omit if download_localhost else (download.mode | default(omit)) }}"
--
GitLab