From e03e3c4582602ce7e2c8d7489ddf1957ad435f12 Mon Sep 17 00:00:00 2001
From: Victor Morales <v.morales@samsung.com>
Date: Fri, 23 Oct 2020 03:07:46 -0700
Subject: [PATCH] Add Kata Containers support to CRI-O runtime (#6830)
* Enable Kata Containers for CRI-O runtime
Kata Containers is an OCI runtime where containers are run inside
lightweight VMs. This runtime has been enabled for containerd runtime
thru the kata_containers_enabled variable. This change enables Kata
Containers to CRI-O container runtime.
Signed-off-by: Victor Morales <v.morales@samsung.com>
* Set appropiate conmon_cgroup when crio_cgroup_manager is 'cgroupfs'
* Set manage_ns_lifecycle=true when KataContainers is enabed
* Add preinstall check for katacontainers
Signed-off-by: Victor Morales <v.morales@samsung.com>
Co-authored-by: Pasquale Toscano <pasqualetoscano90@gmail.com>
---
.../container-engine/cri-o/defaults/main.yml | 23 +++++++++++++-
roles/container-engine/cri-o/tasks/main.yaml | 6 ++++
.../cri-o/templates/crio.conf.j2 | 30 +++++++++----------
roles/container-engine/cri-o/vars/debian.yml | 7 ++++-
roles/container-engine/cri-o/vars/ubuntu.yml | 7 ++++-
.../preinstall/tasks/0020-verify-settings.yml | 6 ++++
roles/kubespray-defaults/defaults/main.yaml | 2 +-
7 files changed, 61 insertions(+), 20 deletions(-)
diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml
index b76bf27ea..a2d690b3f 100644
--- a/roles/container-engine/cri-o/defaults/main.yml
+++ b/roles/container-engine/cri-o/defaults/main.yml
@@ -11,7 +11,6 @@ crio_pause_image: "{{ pod_infra_image_repo }}:{{ pod_infra_version }}"
# By default unqualified images are not allowed for security reasons
crio_registries: []
-crio_runc_path: "/usr/bin/runc"
crio_seccomp_profile: ""
crio_selinux: "{{ (preinstall_selinux_state == 'enforcing')|lower }}"
crio_signature_policy: "{% if ansible_os_family == 'ClearLinux' %}/usr/share/defaults/crio/policy.json{% endif %}"
@@ -29,3 +28,25 @@ crio_kubernetes_version_matrix:
"1.17": "1.17"
crio_version: "{{ crio_kubernetes_version_matrix[crio_required_version] | default('1.19') }}"
+
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+ - name: runc
+ path: /usr/bin/runc
+ type: oci
+ root: /run/runc
+
+# Kata Containers is an OCI runtime, where containers are run inside lightweight
+# VMs. Kata provides additional isolation towards the host, minimizing the host attack
+# surface and mitigating the consequences of containers breakout.
+kata_runtimes:
+ # Kata Containers with the default configured VMM
+ - name: kata-runtime
+ path: /opt/kata/bin/kata-runtime
+ type: oci
+ root: /run/kata-containers
+ # Kata Containers with the QEMU VMM
+ - name: kata-qemu
+ path: /opt/kata/bin/kata-qemu
+ type: oci
+ root: /run/kata-containers
diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml
index 504330d13..61bea7af6 100644
--- a/roles/container-engine/cri-o/tasks/main.yaml
+++ b/roles/container-engine/cri-o/tasks/main.yaml
@@ -29,6 +29,12 @@
- import_tasks: "crictl.yml"
+- name: Build a list of crio runtimes
+ set_fact:
+ crio_runtimes: "{{ crio_runtimes + kata_runtimes }}"
+ when:
+ - kata_containers_enabled
+
- name: Make sure needed folders exist in the system
with_items:
- /etc/crio
diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2
index fd85e5eb4..a456d16d4 100644
--- a/roles/container-engine/cri-o/templates/crio.conf.j2
+++ b/roles/container-engine/cri-o/templates/crio.conf.j2
@@ -110,7 +110,11 @@ decryption_keys_path = "/etc/crio/keys/"
conmon = "{{ crio_conmon }}"
# Cgroup setting for conmon
+{% if crio_cgroup_manager == "cgroupfs" %}
+conmon_cgroup = "pod"
+{% else %}
conmon_cgroup = "system.slice"
+{% endif %}
# Environment variable list for the conmon process, used for passing necessary
# environment variables to conmon or the runtime.
@@ -250,7 +254,11 @@ ctr_stop_timeout = 30
# manage_ns_lifecycle determines whether we pin and remove namespaces
# and manage their lifecycle
+{% if kata_containers_enabled %}
+manage_ns_lifecycle = true
+{% else %}
manage_ns_lifecycle = false
+{% endif %}
# The directory where the state of the managed namespaces gets tracked.
# Only used when manage_ns_lifecycle is true.
@@ -280,22 +288,12 @@ pinns_path = ""
# - runtime_root (optional, string): root directory for storage of containers
# state.
-
-[crio.runtime.runtimes.runc]
-runtime_path = "{{ crio_runc_path }}"
-runtime_type = "oci"
-runtime_root = "/run/runc"
-
-
-# Kata Containers is an OCI runtime, where containers are run inside lightweight
-# VMs. Kata provides additional isolation towards the host, minimizing the host attack
-# surface and mitigating the consequences of containers breakout.
-
-# Kata Containers with the default configured VMM
-#[crio.runtime.runtimes.kata-runtime]
-
-# Kata Containers with the QEMU VMM
-#[crio.runtime.runtimes.kata-qemu]
+{% for runtime in crio_runtimes %}
+[crio.runtime.runtimes.{{ runtime.name }}]
+runtime_path = "{{ runtime.path }}"
+runtime_type = "{{ runtime.type }}"
+runtime_root = "{{ runtime.root }}"
+{% endfor %}
# Kata Containers with the Firecracker VMM
#[crio.runtime.runtimes.kata-fc]
diff --git a/roles/container-engine/cri-o/vars/debian.yml b/roles/container-engine/cri-o/vars/debian.yml
index 0ba558e02..cc9ca8442 100644
--- a/roles/container-engine/cri-o/vars/debian.yml
+++ b/roles/container-engine/cri-o/vars/debian.yml
@@ -4,4 +4,9 @@ crio_packages:
- "cri-o"
- "cri-o-runc"
-crio_runc_path: /usr/sbin/runc
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+ - name: runc
+ path: /usr/sbin/runc
+ type: oci
+ root: /run/runc
diff --git a/roles/container-engine/cri-o/vars/ubuntu.yml b/roles/container-engine/cri-o/vars/ubuntu.yml
index 0ba558e02..cc9ca8442 100644
--- a/roles/container-engine/cri-o/vars/ubuntu.yml
+++ b/roles/container-engine/cri-o/vars/ubuntu.yml
@@ -4,4 +4,9 @@ crio_packages:
- "cri-o"
- "cri-o-runc"
-crio_runc_path: /usr/sbin/runc
+# The crio_runtimes variable defines a list of OCI compatible runtimes.
+crio_runtimes:
+ - name: runc
+ path: /usr/sbin/runc
+ type: oci
+ root: /run/runc
diff --git a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
index e590d6609..7157299c8 100644
--- a/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
+++ b/roles/kubernetes/preinstall/tasks/0020-verify-settings.yml
@@ -286,6 +286,12 @@
msg: "download_run_once support only for docker. See https://github.com/containerd/containerd/issues/4075 for details"
when: download_run_once or download_force_cache
+- name: Stop if kata_containers_enabled is enabled when container_manager is docker
+ assert:
+ that: container_manager != 'docker'
+ msg: "kata_containers_enabled support only for containerd and crio-o. See https://github.com/kata-containers/documentation/blob/1.11.4/how-to/run-kata-with-k8s.md#install-a-cri-implementation for details"
+ when: kata_containers_enabled
+
- name: Stop if download_localhost is enabled for Flatcar Container Linux
assert:
that: ansible_os_family not in ["Flatcar Container Linux by Kinvolk"]
diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml
index 7a9c31dc2..b412b7da5 100644
--- a/roles/kubespray-defaults/defaults/main.yaml
+++ b/roles/kubespray-defaults/defaults/main.yaml
@@ -221,7 +221,7 @@ kube_profiling: false
container_manager: docker
# Enable Kata Containers as additional container runtime
-# When enabled, it requires container_manager=containerd
+# When enabled, it requires `container_manager` different than Docker
kata_containers_enabled: false
# Container on localhost (download images when download_localhost is true)
--
GitLab