diff --git a/docs/ci-setup.md b/docs/ci-setup.md index 604706c798970b522f7192735d878119a521a0e7..a9d380eab75cfcd65b8b31557e60e9e83587ae4e 100644 --- a/docs/ci-setup.md +++ b/docs/ci-setup.md @@ -80,10 +80,15 @@ docker_registry_mirrors: containerd_grpc_max_recv_message_size: 16777216 containerd_grpc_max_send_message_size: 16777216 -containerd_registries: - "docker.io": - - "https://mirror.gcr.io" - - "https://registry-1.docker.io" +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://mirror.gcr.io + capabilities: ["pull", "resolve"] + skip_verify: false + - host: https://registry-1.docker.io + capabilities: ["pull", "resolve"] + skip_verify: false containerd_max_container_log_line_size: -1 diff --git a/docs/containerd.md b/docs/containerd.md index 97917258beec4fea593b75dff1f1e83c024c3c01..c911a8df40418cb09d39c40525a1dfdd6a014027 100644 --- a/docs/containerd.md +++ b/docs/containerd.md @@ -24,15 +24,20 @@ etcd_deployment_type: host Example: define registry mirror for docker hub ```yaml -containerd_registries: - "docker.io": - - "https://mirror.gcr.io" - - "https://registry-1.docker.io" +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://mirror.gcr.io + capabilities: ["pull", "resolve"] + skip_verify: false + - host: https://registry-1.docker.io + capabilities: ["pull", "resolve"] + skip_verify: false ``` -`containerd_registries` is ignored for pulling images when `image_command_tool=nerdctl` +`containerd_registries_mirrors` is ignored for pulling images when `image_command_tool=nerdctl` (the default for `container_manager=containerd`). Use `crictl` instead, it supports -`containerd_registries` but lacks proper multi-arch support (see +`containerd_registries_mirrors` but lacks proper multi-arch support (see [#8375](https://github.com/kubernetes-sigs/kubespray/issues/8375)): ```yaml @@ -103,10 +108,22 @@ containerd_runc_runtime: Config insecure-registry access to self hosted registries. ```yaml -containerd_insecure_registries: - "test.registry.io": "http://test.registry.io" - "172.19.16.11:5000": "http://172.19.16.11:5000" - "repo:5000": "http://repo:5000" +containerd_registries_mirrors: + - prefix: test.registry.io + mirrors: + - host: http://test.registry.io + capabilities: ["pull", "resolve"] + skip_verify: true + - prefix: 172.19.16.11:5000 + mirrors: + - host: http://172.19.16.11:5000 + capabilities: ["pull", "resolve"] + skip_verify: true + - prefix: repo:5000 + mirrors: + - host: http://repo:5000 + capabilities: ["pull", "resolve"] + skip_verify: true ``` [containerd]: https://containerd.io/ diff --git a/docs/offline-environment.md b/docs/offline-environment.md index 692e94d45f9722b0618999d647951f589805c1a7..743c2f7415f0b5a66ec0d7334c18b4f73dc207f2 100644 --- a/docs/offline-environment.md +++ b/docs/offline-environment.md @@ -51,8 +51,12 @@ containerd_download_url: "{{ files_repo }}/containerd-{{ containerd_version }}-l runc_download_url: "{{ files_repo }}/runc.{{ image_arch }}" nerdctl_download_url: "{{ files_repo }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" # Insecure registries for containerd -containerd_insecure_registries: - "{{ registry_addr }}":"{{ registry_host }}" +containerd_registries_mirrors: + - prefix: "{{ registry_addr }}" + mirrors: + - host: "{{ registry_host }}" + capabilities: ["pull", "resolve"] + skip_verify: true # CentOS/Redhat/AlmaLinux/Rocky Linux ## Docker / Containerd diff --git a/inventory/sample/group_vars/all/containerd.yml b/inventory/sample/group_vars/all/containerd.yml index 78ed6636e47323981f2212bd4f447e8c637657b5..1888b24180d27596ce243d1234d43097911884b0 100644 --- a/inventory/sample/group_vars/all/containerd.yml +++ b/inventory/sample/group_vars/all/containerd.yml @@ -30,17 +30,13 @@ # containerd_metrics_grpc_histogram: false -## An obvious use case is allowing insecure-registry access to self hosted registries. -## Can be ipaddress and domain_name. -## example define mirror.registry.io or 172.19.16.11:5000 -## set "name": "url". insecure url must be started http:// -## Port number is also needed if the default HTTPS port is not used. -# containerd_insecure_registries: -# "localhost": "http://127.0.0.1" -# "172.19.16.11:5000": "http://172.19.16.11:5000" - -# containerd_registries: -# "docker.io": "https://registry-1.docker.io" +# Registries defined within containerd. +# containerd_registries_mirrors: +# - prefix: docker.io +# mirrors: +# - host: https://registry-1.docker.io +# capabilities: ["pull", "resolve"] +# skip_verify: false # containerd_max_container_log_line_size: -1 diff --git a/roles/container-engine/containerd/defaults/main.yml b/roles/container-engine/containerd/defaults/main.yml index 4c2df2aba8907d61742ec3145f8b4301711d2846..05cfd95c1894739f2899ebc8a277ae9b6fcff8f0 100644 --- a/roles/container-engine/containerd/defaults/main.yml +++ b/roles/container-engine/containerd/defaults/main.yml @@ -50,6 +50,13 @@ containerd_metrics_grpc_histogram: false containerd_registries: "docker.io": "https://registry-1.docker.io" +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://registry-1.docker.io + capabilities: ["pull", "resolve"] + skip_verify: false + containerd_max_container_log_line_size: -1 # If enabled it will allow non root users to use port numbers <1024 @@ -74,7 +81,7 @@ containerd_limit_core: "infinity" containerd_limit_open_file_num: "infinity" containerd_limit_mem_lock: "infinity" -# If enabled it will use config_path and disable use mirrors config +# If enabled it will use config_path and config to be put in {{ containerd_cfg_dir }}/certs.d/ containerd_use_config_path: false # OS distributions that already support containerd diff --git a/roles/container-engine/containerd/tasks/main.yml b/roles/container-engine/containerd/tasks/main.yml index cab15ffdc711dbc0290ff224a8ac3a13ae3885ae..43aa689526d8e96a87775e50000140778b59498c 100644 --- a/roles/container-engine/containerd/tasks/main.yml +++ b/roles/container-engine/containerd/tasks/main.yml @@ -112,26 +112,20 @@ notify: Restart containerd - name: Containerd | Configure containerd registries - when: containerd_use_config_path is defined and containerd_use_config_path | bool and containerd_insecure_registries is defined + when: containerd_registries_mirrors is defined block: - - name: Containerd | Create registry directories + - name: Containerd | Create registry directories file: - path: "{{ containerd_cfg_dir }}/certs.d/{{ item.key }}" + path: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}" state: directory mode: 0755 - recurse: true - with_dict: "{{ containerd_insecure_registries }}" - - name: Containerd | Write hosts.toml file - blockinfile: - path: "{{ containerd_cfg_dir }}/certs.d/{{ item.key }}/hosts.toml" + loop: "{{ containerd_registries_mirrors }}" + - name: Containerd | Write hosts.toml file + template: + src: hosts.toml.j2 + dest: "{{ containerd_cfg_dir }}/certs.d/{{ item.prefix }}/hosts.toml" mode: 0640 - create: true - block: | - server = "{{ item.value }}" - [host."{{ item.value }}"] - capabilities = ["pull", "resolve", "push"] - skip_verify = true - with_dict: "{{ containerd_insecure_registries }}" + loop: "{{ containerd_registries_mirrors }}" # you can sometimes end up in a state where everything is installed # but containerd was not started / enabled diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index e47069ecb3e381f75f4676a871109ba96f5855d2..a04ec485e5ea8f964486c4d9d0f88cb8b313c8d5 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -51,18 +51,18 @@ oom_score = {{ containerd_oom_score }} config_path = "{{ containerd_cfg_dir }}/certs.d" {% else %} [plugins."io.containerd.grpc.v1.cri".registry.mirrors] -{% for registry, addr in containerd_registries.items() %} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"] - endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"] +{% set insecure_registries_addr = [] %} +{% for registry in containerd_registries_mirrors %} + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry.prefix }}"] +{% set endpoint = [] %} +{% for mirror in registry.mirrors %} +{% if endpoint.append(mirror.host) %}{% endif %} +{% if mirror.skip_verify is defined and mirror.skip_verify|bool %}{% if insecure_registries_addr.append(mirror.host | urlsplit('netloc')) %}{% endif %}{% endif %} {% endfor %} -{% if containerd_insecure_registries is defined and containerd_insecure_registries|length>0 %} -{% for registry, addr in containerd_insecure_registries.items() %} - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry }}"] - endpoint = ["{{ ([ addr ] | flatten ) | join('","') }}"] + endpoint = ["{{ ( endpoint | unique ) | join('","') }}"] {% endfor %} -{% endif %} -{% for addr in containerd_insecure_registries.values() | flatten | unique %} - [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr | urlsplit('netloc') }}".tls] +{% for addr in insecure_registries_addr | unique %} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ addr }}".tls] insecure_skip_verify = true {% endfor %} {% endif %} diff --git a/roles/container-engine/containerd/templates/hosts.toml.j2 b/roles/container-engine/containerd/templates/hosts.toml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..c04dc478995696793498042a86861d256b28b44d --- /dev/null +++ b/roles/container-engine/containerd/templates/hosts.toml.j2 @@ -0,0 +1,8 @@ +server = "https://{{ item.prefix }}" +{% for mirror in item.mirrors %} +[host."{{ mirror.host }}"] + capabilities = ["{{ ([ mirror.capabilities ] | flatten ) | join('","') }}"] +{% if mirror.skip_verify is defined %} + skip_verify = {{ mirror.skip_verify | default('false') | string | lower }} +{% endif %} +{% endfor %} diff --git a/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 b/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 index 7fed7a00131d88cfc402df18eb3a252215268a0d..8b590f6f53c69a09d88b7e320894c7784a89f3d1 100644 --- a/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 +++ b/roles/container-engine/nerdctl/templates/nerdctl.toml.j2 @@ -6,5 +6,4 @@ snapshotter = "{{ nerdctl_snapshotter | default('overlayfs') }}" cni_path = "/opt/cni/bin" cni_netconfpath = "/etc/cni/net.d" cgroup_manager = "{{ kubelet_cgroup_driver | default('systemd') }}" -insecure_registry = {{ (containerd_insecure_registries is defined and containerd_insecure_registries|length>0) | bool | lower }} -hosts_dir = ["/etc/containerd/certs.d"] +hosts_dir = ["{{ containerd_cfg_dir }}/certs.d"] diff --git a/roles/download/defaults/main/main.yml b/roles/download/defaults/main/main.yml index c6542c65800dcad863c29b64bfd90166fe393c0e..c0430d82ca74dd6854d44a80d36e5ed530f2c82c 100644 --- a/roles/download/defaults/main/main.yml +++ b/roles/download/defaults/main/main.yml @@ -57,7 +57,7 @@ download_retries: 4 docker_image_pull_command: "{{ docker_bin_dir }}/docker pull" docker_image_info_command: "{{ docker_bin_dir }}/docker images -q | xargs -i {{ '{{' }} docker_bin_dir }}/docker inspect -f {% raw %}'{{ '{{' }} if .RepoTags }}{{ '{{' }} join .RepoTags \",\" }}{{ '{{' }} end }}{{ '{{' }} if .RepoDigests }},{{ '{{' }} join .RepoDigests \",\" }}{{ '{{' }} end }}' {% endraw %} {} | tr '\n' ','" nerdctl_image_info_command: "{{ bin_dir }}/nerdctl -n k8s.io images --format '{% raw %}{{ .Repository }}:{{ .Tag }}{% endraw %}' 2>/dev/null | grep -v ^:$ | tr '\n' ','" -nerdctl_image_pull_command: "{{ bin_dir }}/nerdctl -n k8s.io pull --quiet {{ nerdctl_extra_flags }}" +nerdctl_image_pull_command: "{{ bin_dir }}/nerdctl -n k8s.io pull --quiet" crictl_image_info_command: "{{ bin_dir }}/crictl images --verbose | awk -F ': ' '/RepoTags|RepoDigests/ {print $2}' | tr '\n' ','" crictl_image_pull_command: "{{ bin_dir }}/crictl pull" @@ -72,9 +72,6 @@ image_info_command_on_localhost: "{{ lookup('vars', image_command_tool_on_localh # Arch of Docker images and needed packages image_arch: "{{ host_architecture | default('amd64') }}" -# Nerdctl insecure flag set -nerdctl_extra_flags: '{%- if containerd_insecure_registries is defined and containerd_insecure_registries | length > 0 -%}--insecure-registry{%- else -%}{%- endif -%}' - # Versions kubeadm_version: "{{ kube_version }}" crun_version: 1.8.5 diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 9e56eacef075ce3cc534aa6ef9f824c7622306e5..fb279f130dd499bd5ea87e32112c93d57c0c3447 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -356,15 +356,6 @@ docker_plugins: [] # Containerd options - thse are relevant when container_manager == 'containerd' containerd_use_systemd_cgroup: true -## An obvious use case is allowing insecure-registry access to self hosted registries. -## Can be ipaddress and domain_name. -## example define mirror.registry.io or 172.19.16.11:5000 -## Port number is also needed if the default HTTPS port is not used. -# containerd_insecure_registries: -# "mirror.registry.io":"http://mirror.registry.io" -# "172.19.16.11:5000":"http://172.19.16.11:5000" -containerd_insecure_registries: {} - # Containerd conf default dir containerd_storage_dir: "/var/lib/containerd" containerd_state_dir: "/run/containerd" diff --git a/tests/common/_docker_hub_registry_mirror.yml b/tests/common/_docker_hub_registry_mirror.yml index e6298b70ed8f05e337f2bfc2333a2c64c5859dcb..db521d67952a651e844d0c1b76ac028e99ec262d 100644 --- a/tests/common/_docker_hub_registry_mirror.yml +++ b/tests/common/_docker_hub_registry_mirror.yml @@ -10,6 +10,16 @@ containerd_registries: - "https://mirror.gcr.io" - "https://registry-1.docker.io" +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://mirror.gcr.io + capabilities: ["pull", "resolve"] + skip_verify: false + - host: https://registry-1.docker.io + capabilities: ["pull", "resolve"] + skip_verify: false + containerd_max_container_log_line_size: -1 crio_registries: diff --git a/tests/files/packet_ubuntu22-calico-aio.yml b/tests/files/packet_ubuntu22-calico-aio.yml index abe2d50f13339d161220e36465c30a88c2914683..c9458f5689f3eff98e1f603667a3eab55de1dd00 100644 --- a/tests/files/packet_ubuntu22-calico-aio.yml +++ b/tests/files/packet_ubuntu22-calico-aio.yml @@ -11,8 +11,17 @@ auto_renew_certificates: true kube_proxy_mode: iptables enable_nodelocaldns: False -containerd_insecure_registries: - "172.19.16.11:5000": "http://172.19.16.11:5000" - containerd_registries: - "docker.io": "https://mirror.gcr.io" \ No newline at end of file + "docker.io": "https://mirror.gcr.io" + +containerd_registries_mirrors: + - prefix: docker.io + mirrors: + - host: https://mirror.gcr.io + capabilities: ["pull", "resolve"] + skip_verify: false + - prefix: 172.19.16.11:5000 + mirrors: + - host: http://172.19.16.11:5000 + capabilities: ["pull", "resolve", "push"] + skip_verify: true