diff --git a/docs/_sidebar.md b/docs/_sidebar.md index e07deab4eef1de34597d0e5fedaa129d500b6c15..d704e8b448eadbc2de150c9b53dff1272f7f6cf1 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -41,6 +41,7 @@ * [Containerd](docs/containerd.md) * [CRI-O](docs/cri-o.md) * [Kata Containers](docs/kata-containers.md) + * [gVisor](docs/gvisor.md) * Advanced * [Proxy](/docs/proxy.md) * [Downloads](docs/downloads.md) diff --git a/docs/gvisor.md b/docs/gvisor.md new file mode 100644 index 0000000000000000000000000000000000000000..ef0a64b108166a84c3907f4c371a8c54e65e0c72 --- /dev/null +++ b/docs/gvisor.md @@ -0,0 +1,16 @@ +# gVisor + +[gVisor](https://gvisor.dev/docs/) is an application kernel, written in Go, that implements a substantial portion of the Linux system call interface. It provides an additional layer of isolation between running applications and the host operating system. + +gVisor includes an Open Container Initiative (OCI) runtime called runsc that makes it easy to work with existing container tooling. The runsc runtime integrates with Docker and Kubernetes, making it simple to run sandboxed containers. + +## Usage + +To enable gVisor you should be using a container manager that is compatible with selecting the [RuntimeClass](https://kubernetes.io/docs/concepts/containers/runtime-class/) such as `containerd`. + +Containerd support: + +```yaml +container_manager: containerd +gvisor_enabled: true +``` diff --git a/roles/container-engine/containerd-common/defaults/main.yml b/roles/container-engine/containerd-common/defaults/main.yml index 5ee766409c7644fda5f000aac76254d82cbcdbd2..e1555e986d3ce9e3a181311811747bd645a6925b 100644 --- a/roles/container-engine/containerd-common/defaults/main.yml +++ b/roles/container-engine/containerd-common/defaults/main.yml @@ -1,2 +1,17 @@ --- containerd_package: 'containerd.io' + +# Fedora docker-ce repo +docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg' +# CentOS/RedHat docker-ce repo +docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable' +docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' +# Ubuntu docker-ce repo +docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" +docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' +docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' +# Debian docker-ce repo +docker_debian_repo_base_url: "https://download.docker.com/linux/debian" +docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' +docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' diff --git a/roles/container-engine/containerd/templates/config.toml.j2 b/roles/container-engine/containerd/templates/config.toml.j2 index 8fca568143589fe228086f7254b22f53cfffc3c3..a344e4b96b6b5f42fd624c18e8f82fdc660f054f 100644 --- a/roles/container-engine/containerd/templates/config.toml.j2 +++ b/roles/container-engine/containerd/templates/config.toml.j2 @@ -35,6 +35,10 @@ oom_score = {{ containerd_oom_score }} {% if kata_containers_enabled %} [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-qemu] runtime_type = "io.containerd.kata-qemu.v2" +{% endif %} +{% if gvisor_enabled %} + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runsc] + runtime_type = "io.containerd.runsc.v1" {% endif %} [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] diff --git a/roles/container-engine/docker/defaults/main.yml b/roles/container-engine/docker/defaults/main.yml index 09cdbb6a85b23eebd7d0a051964b673052ebd988..5f376b03e29303fb67e38b8b2e48e76830a19b74 100644 --- a/roles/container-engine/docker/defaults/main.yml +++ b/roles/container-engine/docker/defaults/main.yml @@ -15,20 +15,6 @@ docker_cgroup_driver: systemd yum_repo_dir: /etc/yum.repos.d -# Fedora docker-ce repo -docker_fedora_repo_base_url: 'https://download.docker.com/linux/fedora/{{ ansible_distribution_major_version }}/$basearch/stable' -docker_fedora_repo_gpgkey: 'https://download.docker.com/linux/fedora/gpg' -# CentOS/RedHat docker-ce repo -docker_rh_repo_base_url: 'https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable' -docker_rh_repo_gpgkey: 'https://download.docker.com/linux/centos/gpg' -# Ubuntu docker-ce repo -docker_ubuntu_repo_base_url: "https://download.docker.com/linux/ubuntu" -docker_ubuntu_repo_gpgkey: 'https://download.docker.com/linux/ubuntu/gpg' -docker_ubuntu_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' -# Debian docker-ce repo -docker_debian_repo_base_url: "https://download.docker.com/linux/debian" -docker_debian_repo_gpgkey: 'https://download.docker.com/linux/debian/gpg' -docker_debian_repo_repokey: '9DC858229FC7DD38854AE2D88D81803C0EBFCD88' docker_bin_dir: "/usr/bin" # flag to enable/disable docker cleanup diff --git a/roles/container-engine/gvisor/molecule/default/converge.yml b/roles/container-engine/gvisor/molecule/default/converge.yml new file mode 100644 index 0000000000000000000000000000000000000000..8bf5478e870ae5d3664277f44ad497ebc23a55cf --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/converge.yml @@ -0,0 +1,11 @@ +--- +- name: Converge + hosts: all + become: true + vars: + gvisor_enabled: true + container_manager: containerd + roles: + - role: kubespray-defaults + - role: containerd + - role: gvisor diff --git a/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf b/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf new file mode 100644 index 0000000000000000000000000000000000000000..f10935b753b7c53037b66c0e6960096216a5abc8 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/10-mynet.conf @@ -0,0 +1,17 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "172.19.0.0/24", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ] + } +} diff --git a/roles/container-engine/gvisor/molecule/default/files/container.json b/roles/container-engine/gvisor/molecule/default/files/container.json new file mode 100644 index 0000000000000000000000000000000000000000..3490359d0f34232ddbc622c5e0a7702cc0157f2e --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/container.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1" + }, + "image": { + "image": "docker.io/library/hello-world:latest" + }, + "log_path": "gvisor1.0.log", + "linux": {} +} diff --git a/roles/container-engine/gvisor/molecule/default/files/sandbox.json b/roles/container-engine/gvisor/molecule/default/files/sandbox.json new file mode 100644 index 0000000000000000000000000000000000000000..a8da54d91db75a809bb02e42eb6f6140fb24ecb0 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/files/sandbox.json @@ -0,0 +1,10 @@ +{ + "metadata": { + "name": "gvisor1", + "namespace": "default", + "attempt": 1, + "uid": "hdishd83djaidwnduwk28bcsb" + }, + "linux": {}, + "log_directory": "/tmp" +} diff --git a/roles/container-engine/gvisor/molecule/default/molecule.yml b/roles/container-engine/gvisor/molecule/default/molecule.yml new file mode 100644 index 0000000000000000000000000000000000000000..fc4ec0276bafc06cfa1cd540b21ef60d91db6f6e --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/molecule.yml @@ -0,0 +1,44 @@ +--- +driver: + name: vagrant + provider: + name: libvirt + options: + driver: kvm +lint: | + set -e + yamllint -c ../../../.yamllint . +platforms: + - name: ubuntu20 + box: generic/ubuntu2004 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane + - name: centos8 + box: generic/centos8 + cpus: 1 + memory: 1024 + nested: true + groups: + - kube_control_plane +provisioner: + name: ansible + env: + ANSIBLE_ROLES_PATH: ../../../../ + config_options: + defaults: + callback_whitelist: profile_tasks + lint: + name: ansible-lint + options: + c: ../../../.ansible-lint + inventory: + group_vars: + all: + become: true +verifier: + name: testinfra + lint: + name: flake8 diff --git a/roles/container-engine/gvisor/molecule/default/prepare.yml b/roles/container-engine/gvisor/molecule/default/prepare.yml new file mode 100644 index 0000000000000000000000000000000000000000..084824830cc04bbea76e989c12375be101e0cef1 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/prepare.yml @@ -0,0 +1,48 @@ +--- +- name: Prepare generic + hosts: all + become: true + roles: + - role: kubespray-defaults + - role: bootstrap-os + - role: ../adduser + user: "{{ addusers.kube }}" + tasks: + - include_tasks: "../../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.cni) }}" + +- name: Prepare container runtime + hosts: all + become: true + vars: + container_manager: containerd + kube_network_plugin: cni + roles: + - role: kubespray-defaults + - role: ../network_plugin/cni + - role: crictl + tasks: + - name: Copy test container files + copy: + src: "{{ item }}" + dest: "/tmp/{{ item }}" + owner: root + mode: 0644 + with_items: + - container.json + - sandbox.json + - name: Create /etc/cni/net.d directory + file: + path: /etc/cni/net.d + state: directory + owner: kube + mode: 0755 + - name: Setup CNI + copy: + src: "{{ item }}" + dest: "/etc/cni/net.d/{{ item }}" + owner: root + mode: 0644 + with_items: + - 10-mynet.conf diff --git a/roles/container-engine/gvisor/molecule/default/tests/test_default.py b/roles/container-engine/gvisor/molecule/default/tests/test_default.py new file mode 100644 index 0000000000000000000000000000000000000000..eb40b00cb3a144805be21ea8698092324a214401 --- /dev/null +++ b/roles/container-engine/gvisor/molecule/default/tests/test_default.py @@ -0,0 +1,29 @@ +import os + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') + + +def test_run(host): + gvisorruntime = "/usr/local/bin/runsc" + with host.sudo(): + cmd = host.command(gvisorruntime + " --version") + assert cmd.rc == 0 + assert "runsc version" in cmd.stdout + + +def test_run_pod(host): + runtime = "runsc" + + run_command = "/usr/local/bin/crictl run --with-pull --runtime {} /tmp/container.json /tmp/sandbox.json".format(runtime) + with host.sudo(): + cmd = host.command(run_command) + assert cmd.rc == 0 + + with host.sudo(): + log_f = host.file("/tmp/gvisor1.0.log") + + assert log_f.exists + assert b"Hello from Docker!" in log_f.content diff --git a/roles/container-engine/gvisor/tasks/main.yml b/roles/container-engine/gvisor/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..fa5bd725e4ed93f7abd160631247eea4b5dc34bd --- /dev/null +++ b/roles/container-engine/gvisor/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: gVisor | Download runsc binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_runsc) }}" + +- name: gVisor | Download containerd-shim-runsc-v1 binary + include_tasks: "../../../download/tasks/download_file.yml" + vars: + download: "{{ download_defaults | combine(downloads.gvisor_containerd_shim) }}" + +- name: gVisor | Copy binaries + copy: + src: "{{ local_release_dir }}/gvisor-{{ item }}" + dest: "{{ bin_dir }}/{{ item }}" + mode: 0755 + remote_src: yes + with_items: + - runsc + - containerd-shim-runsc-v1 diff --git a/roles/container-engine/meta/main.yml b/roles/container-engine/meta/main.yml index 6218db932f4a0dfae4cd8471f73f7eb7638f6bbe..2f6bff147e5d234cdab16b86c5b4dc8c0866b975 100644 --- a/roles/container-engine/meta/main.yml +++ b/roles/container-engine/meta/main.yml @@ -7,6 +7,14 @@ dependencies: - container-engine - kata-containers + - role: container-engine/gvisor + when: + - gvisor_enabled + - container_manager in ['docker', 'containerd'] + tags: + - container-engine + - gvisor + - role: container-engine/crun when: - crun_enabled diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index afcfc944f961eca6fdf8e9a18ebbb7cbe066ccdb..7b465384d5754fabbaf4d928b4e216b1645f7c55 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -53,6 +53,7 @@ kubeadm_version: "{{ kube_version }}" etcd_version: v3.4.13 crun_version: 0.19 kata_containers_version: 2.1.0 +gvisor_version: 20210518.0 # gcr and kubernetes image repo define gcr_image_repo: "gcr.io" @@ -108,6 +109,8 @@ crictl_download_url: "https://github.com/kubernetes-sigs/cri-tools/releases/down helm_download_url: "https://get.helm.sh/helm-{{ helm_version }}-linux-{{ image_arch }}.tar.gz" crun_download_url: "https://github.com/containers/crun/releases/download/{{ crun_version }}/crun-{{ crun_version }}-linux-{{ image_arch }}" kata_containers_download_url: "https://github.com/kata-containers/kata-containers/releases/download/{{ kata_containers_version }}/kata-static-{{ kata_containers_version }}-{{ ansible_architecture }}.tar.xz" +# gVisor only supports amd64 and uses x86_64 to in the download link +gvisor_download_url: "https://storage.googleapis.com/gvisor/releases/release/{{ gvisor_version }}/{{ ansible_architecture }}" nerdctl_download_url: "https://github.com/containerd/nerdctl/releases/download/v{{ nerdctl_version }}/nerdctl-{{ nerdctl_version }}-{{ ansible_system | lower }}-{{ image_arch }}.tar.gz" krew_download_url: "https://github.com/kubernetes-sigs/krew/releases/download/{{ krew_version }}/krew.tar.gz" @@ -401,6 +404,22 @@ kata_containers_binary_checksums: 2.0.4: 0 2.1.0: 0 +gvisor_runsc_binary_checksums: + arm: + 20210518.0: 0 + arm64: + 20210518.0: 0 + amd64: + 20210518.0: 907811e10576d31b6f63d2f91f1c3be2b2df94e222fb48eb509fce48cd6ae9e0 + +gvisor_containerd_shim_binary_checksums: + arm: + 20210518.0: 0 + arm64: + 20210518.0: 0 + amd64: + 20210518.0: d6deda79562cadf4fc98ccf1d6d9fd1d2dff3890b184c7b546167bd6eff241ec + nerdctl_archive_checksums: arm: 0.8.1: 27bdad3f9e2667620f70617c48d595c5c4e24a10fbcd00d31202cd8d571c9233 @@ -420,6 +439,8 @@ crictl_binary_checksum: "{{ crictl_checksums[image_arch][crictl_version] }}" helm_archive_checksum: "{{ helm_archive_checksums[image_arch][helm_version] }}" crun_binary_checksum: "{{ crun_checksums[image_arch][crun_version] }}" kata_containers_binary_checksum: "{{ kata_containers_binary_checksums[image_arch][kata_containers_version] }}" +gvisor_runsc_binary_checksum: "{{ gvisor_runsc_binary_checksums[image_arch][gvisor_version] }}" +gvisor_containerd_shim_binary_checksum: "{{ gvisor_containerd_shim_binary_checksums[image_arch][gvisor_version] }}" nerdctl_archive_checksum: "{{ nerdctl_archive_checksums[image_arch][nerdctl_version] }}" krew_archive_checksum: "{{ krew_archive_checksums[krew_version] }}" @@ -715,6 +736,32 @@ downloads: groups: - k8s_cluster + gvisor_runsc: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-runsc" + sha256: "{{ gvisor_runsc_binary_checksum }}" + url: "{{ gvisor_download_url }}/runsc" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + + gvisor_containerd_shim: + enabled: "{{ gvisor_enabled }}" + file: true + version: "{{ gvisor_version }}" + dest: "{{ local_release_dir }}/gvisor-containerd-shim-runsc-v1" + sha256: "{{ gvisor_containerd_shim_binary_checksum }}" + url: "{{ gvisor_download_url }}/containerd-shim-runsc-v1" + unarchive: false + owner: "root" + mode: 755 + groups: + - k8s_cluster + nerdctl: file: true enabled: "{{ nerdctl_enabled }}" diff --git a/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml b/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f12f5feae8963f9d050ec9d0107ff1bf69c2425f --- /dev/null +++ b/roles/kubernetes-apps/container_runtimes/gvisor/tasks/main.yaml @@ -0,0 +1,33 @@ +--- +- name: gVisor | Create addon dir + file: + path: "{{ kube_config_dir}}/addons/gvisor" + owner: root + group: root + mode: 0755 + recurse: true + +- name: gVisor | Templates List + set_fact: + gvisor_templates: + - { name: runtimeclass-gvisor, file: runtimeclass-gvisor.yml, type: runtimeclass } + +- name: gVisort | Create manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir}}/addons/gvisor/{{ item.file }}" + with_items: "{{ gvisor_templates }}" + register: gvisor_manifests + when: + - inventory_hostname == groups['kube_control_plane'][0] + +- name: gVisor | Apply manifests + kube: + name: "{{ item.item.name }}" + kubectl: "{{ bin_dir}}/kubectl" + resource: "{{ item.item.type }}" + filename: "{{ kube_config_dir }}/addons/gvisor/{{ item.item.file }}" + state: "latest" + with_items: "{{ gvisor_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] diff --git a/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 b/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..92b20b4c6e23e7863750014901fbb5f2461549dd --- /dev/null +++ b/roles/kubernetes-apps/container_runtimes/gvisor/templates/runtimeclass-gvisor.yml.j2 @@ -0,0 +1,6 @@ +--- +kind: RuntimeClass +apiVersion: node.k8s.io/v1{{ 'beta1' if kube_version is version('v1.20.0', '<') else '' }} +metadata: + name: gvisor +handler: runsc diff --git a/roles/kubernetes-apps/container_runtimes/meta/main.yml b/roles/kubernetes-apps/container_runtimes/meta/main.yml index 3c56e992e7d44b7c768afd34f68ddf964f03f2b1..f63d38d618e21c4ec5def7e3b5149e0d257c70c1 100644 --- a/roles/kubernetes-apps/container_runtimes/meta/main.yml +++ b/roles/kubernetes-apps/container_runtimes/meta/main.yml @@ -7,6 +7,13 @@ dependencies: - kata-containers - container-runtimes + - role: kubernetes-apps/container_runtimes/gvisor + when: gvisor_enabled + tags: + - apps + - gvisor + - container-runtimes + - role: kubernetes-apps/container_runtimes/crun when: crun_enabled tags: diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml index ae618243de1313ce399485689f9ac4d484fe1165..0585a11165e90f3e5f5b78238db1c7f1a6dde236 100644 --- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml @@ -312,6 +312,12 @@ msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details" when: kata_containers_enabled +- name: Stop if gvisor_enabled is enabled when container_manager is not containerd + assert: + that: container_manager == 'containerd' + msg: "gvisor_enabled support only compatible with containerd. See https://github.com/kubernetes-sigs/kubespray/issues/7650 for details" + when: gvisor_enabled + - name: Stop if download_localhost is enabled for Flatcar Container Linux assert: that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"] diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 2bcba470f7dde804d59c68635dbac25c47b56971..960aceff35597a4fb899583e24e56d618e3046e9 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -237,6 +237,10 @@ container_manager: docker # When enabled, it requires `container_manager` different than Docker kata_containers_enabled: false +# Enable gVisor as an additional container runtime +# gVisor is only supported with container_manager Docker or containerd +gvisor_enabled: false + # Enable crun as additional container runtime # When enabled, it requires container_manager=crio crun_enabled: false