From 4a8a52bad9ecd89d3879b37601112200a62bfcd9 Mon Sep 17 00:00:00 2001
From: Sergey <s.bondarev@southbridge.ru>
Date: Mon, 30 Nov 2020 11:22:49 +0300
Subject: [PATCH] containerd docker hub registry mirror support (#6962)
* containerd docker hub registry mirror support
* add docs
* fix typo
* fix yamllint
* fix indent in sample
and ansible-playbook param in testcases_run
* fix md
* mv common vars to tests/common/_docker_hub_registry_mirror.yml
* checkout vars to upgrade tests
---
.gitlab-ci.yml | 1 +
docs/containerd.md | 31 +++++++++++++++++++
.../sample/group_vars/all/containerd.yml | 6 +++-
.../containerd/templates/config.toml.j2 | 2 +-
tests/common/_docker_hub_registry_mirror.yml | 15 +++++++++
tests/scripts/testcases_run.sh | 15 ++++-----
6 files changed, 61 insertions(+), 9 deletions(-)
create mode 100644 docs/containerd.md
create mode 100644 tests/common/_docker_hub_registry_mirror.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6d235ddbd..a4f0ceed8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,6 +15,7 @@ variables:
MAGIC: "ci check this"
TEST_ID: "$CI_PIPELINE_ID-$CI_BUILD_ID"
CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
+ CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
GS_ACCESS_KEY_ID: $GS_KEY
GS_SECRET_ACCESS_KEY: $GS_SECRET
CONTAINER_ENGINE: docker
diff --git a/docs/containerd.md b/docs/containerd.md
new file mode 100644
index 000000000..58fd44d8f
--- /dev/null
+++ b/docs/containerd.md
@@ -0,0 +1,31 @@
+# conrainerd
+
+[containerd] An industry-standard container runtime with an emphasis on simplicity, robustness and portability
+Kubespray supports basic functionality for using containerd as the default container runtime in a cluster.
+
+_To use the containerd container runtime set the following variables:_
+
+## k8s-cluster.yml
+
+```yaml
+container_manager: containerd
+```
+
+## Containerd config
+
+Example: define registry mirror for docker hub
+
+```yaml
+containerd_config:
+ grpc:
+ max_recv_message_size: 16777216
+ max_send_message_size: 16777216
+ debug:
+ level: ""
+ registries:
+ "docker.io":
+ - "https://mirror.gcr.io"
+ - "https://registry-1.docker.io"
+```
+
+[containerd]: https://containerd.io/
diff --git a/inventory/sample/group_vars/all/containerd.yml b/inventory/sample/group_vars/all/containerd.yml
index 2fc66b636..0f1e97749 100644
--- a/inventory/sample/group_vars/all/containerd.yml
+++ b/inventory/sample/group_vars/all/containerd.yml
@@ -1,6 +1,8 @@
---
# Please see roles/container-engine/containerd/defaults/main.yml for more configuration options
+# Example: define registry mirror for docker hub
+
# containerd_config:
# grpc:
# max_recv_message_size: 16777216
@@ -8,7 +10,9 @@
# debug:
# level: ""
# registries:
-# "docker.io": "https://registry-1.docker.io"
+# "docker.io":
+# - "https://mirror.gcr.io"
+# - "https://registry-1.docker.io"
# max_container_log_line_size: -1
# metrics:
# address: ""
diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2
index ceccaa2fc..671af29e4 100644
--- a/roles/container-engine/containerd/templates/config.toml.j2
+++ b/roles/container-engine/containerd/templates/config.toml.j2
@@ -62,7 +62,7 @@ disabled_plugins = ["restart"]
[plugins.cri.registry.mirrors]
{% for registry, addr in containerd_config.registries.items() %}
[plugins.cri.registry.mirrors."{{ registry }}"]
- endpoint = ["{{ addr }}"]
+ endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"]
{% endfor %}
{% endif %}
diff --git a/tests/common/_docker_hub_registry_mirror.yml b/tests/common/_docker_hub_registry_mirror.yml
new file mode 100644
index 000000000..3dadb08ff
--- /dev/null
+++ b/tests/common/_docker_hub_registry_mirror.yml
@@ -0,0 +1,15 @@
+---
+docker_registry_mirrors:
+ - "https://mirror.gcr.io"
+
+containerd_config:
+ grpc:
+ max_recv_message_size: 16777216
+ max_send_message_size: 16777216
+ debug:
+ level: ""
+ registries:
+ "docker.io":
+ - "https://mirror.gcr.io"
+ - "https://registry-1.docker.io"
+ max_container_log_line_size: -1
diff --git a/tests/scripts/testcases_run.sh b/tests/scripts/testcases_run.sh
index 3acbfb232..9f9870b57 100755
--- a/tests/scripts/testcases_run.sh
+++ b/tests/scripts/testcases_run.sh
@@ -42,6 +42,7 @@ fi
test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout "$KUBESPRAY_VERSION"
# Checkout the CI vars file so it is available
test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml
+test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" ${CI_TEST_REGISTRY_MIRROR}
# Install mitogen ansible plugin
if [ "${MITOGEN_ENABLE}" = "true" ]; then
@@ -51,20 +52,20 @@ if [ "${MITOGEN_ENABLE}" = "true" ]; then
fi
# Create cluster
-ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" cluster.yml
+ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
# Repeat deployment if testing upgrade
if [ "${UPGRADE_TEST}" != "false" ]; then
test "${UPGRADE_TEST}" == "basic" && PLAYBOOK="cluster.yml"
test "${UPGRADE_TEST}" == "graceful" && PLAYBOOK="upgrade-cluster.yml"
git checkout "${CI_BUILD_REF}"
- ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" $PLAYBOOK
+ ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" $PLAYBOOK
fi
# Test control plane recovery
if [ "${RECOVER_CONTROL_PLANE_TEST}" != "false" ]; then
- ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
- ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
+ ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "${RECOVER_CONTROL_PLANE_TEST_GROUPS}:!fake_hosts" -e reset_confirmation=yes reset.yml
+ ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e etcd_retries=10 --limit etcd,kube-master:!fake_hosts recover-control-plane.yml
fi
# Tests Cases
@@ -88,7 +89,7 @@ ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} --limit "all:!fake_
## Idempotency checks 1/5 (repeat deployment)
if [ "${IDEMPOT_CHECK}" = "true" ]; then
- ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" cluster.yml
+ ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
fi
## Idempotency checks 2/5 (Advanced DNS checks)
@@ -98,12 +99,12 @@ fi
## Idempotency checks 3/5 (reset deployment)
if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
- ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e reset_confirmation=yes -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" reset.yml
+ ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
fi
## Idempotency checks 4/5 (redeploy after reset)
if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
- ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads -e '{"docker_registry_mirrors":["https://mirror.gcr.io"]}' --limit "all:!fake_hosts" cluster.yml
+ ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
fi
## Idempotency checks 5/5 (Advanced DNS checks)
--
GitLab