From 9c3b573f8e5d3504ce82352ec76c1d9ed117f958 Mon Sep 17 00:00:00 2001
From: spaced <spaced.wombat@gmail.com>
Date: Sat, 11 Apr 2020 08:51:47 +0200
Subject: [PATCH] Cleanup fedora coreos with crio container (#5887)

* fix upgrade of crio on fcos
- update documents

* install conntrack required by kube-proxy
- like commit 48c41bcbe7a428584f4919f4404d09a0b7381a53

* enable fedora modular repo for crio

* allow to override crio configuration
- set cgroup manager same to kubelet_cgroup_driver if defined
- path of seccomp_profile depends on distribution

* allow to override crio configuration
- fix path for ubuntu

* allow to override crio configuration
- fix cni path for fcos
---
 README.md                                     |  2 +-
 docs/fcos.md                                  | 24 +++++++++++++++----
 roles/bootstrap-os/defaults/main.yml          |  3 ++-
 .../container-engine/cri-o/defaults/main.yml  |  6 +++++
 roles/container-engine/cri-o/tasks/main.yaml  | 24 ++++++++++++++++---
 .../cri-o/templates/crio.conf.j2              | 20 ++++------------
 .../cri-o/vars/clearlinux.yml                 |  2 ++
 roles/container-engine/cri-o/vars/fedora.yml  |  1 +
 roles/container-engine/cri-o/vars/redhat.yml  |  1 +
 roles/container-engine/cri-o/vars/ubuntu.yml  |  4 +++-
 10 files changed, 60 insertions(+), 27 deletions(-)

diff --git a/README.md b/README.md
index 7d1f28021..57f4b1139 100644
--- a/README.md
+++ b/README.md
@@ -105,7 +105,7 @@ vagrant up
 - **Ubuntu** 16.04, 18.04
 - **CentOS/RHEL** 7, 8 (experimental: see [centos 8 notes](docs/centos8.md)
 - **Fedora** 28
-- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md)
+- **Fedora CoreOS** (experimental: see [fcos Note](docs/fcos.md))
 - **openSUSE** Leap 42.3/Tumbleweed
 - **Oracle Linux** 7
 
diff --git a/docs/fcos.md b/docs/fcos.md
index d31da7373..0682879cc 100644
--- a/docs/fcos.md
+++ b/docs/fcos.md
@@ -1,6 +1,7 @@
 # Fedora CoreOS
 
-Tested with stable version 31.20200223.3.0
+Tested with stable version 31.20200223.3.0.
+
 Because package installation with `rpm-ostree` requires a reboot, playbook may fail while bootstrap.
 Restart playbook again.
 
@@ -35,11 +36,25 @@ systemd:
         WantedBy=multi-user.target
 ```
 
+## Network
+
+### calico
+
+To use calico create sysctl file with ignition:
+
+```yaml
+files:
+    - path: /etc/sysctl.d/reverse-path-filter.conf
+      contents:
+        inline: |
+          net.ipv4.conf.all.rp_filter=1
+```
+
 ## libvirt setup
 
 ### Prepare
 
-Prepare ignition and serve via http (a.e. python -m SimpleHTTPServer )
+Prepare ignition and serve via http (a.e. python -m http.server )
 
 ```json
 {
@@ -50,10 +65,9 @@ Prepare ignition and serve via http (a.e. python -m SimpleHTTPServer )
   "passwd": {
     "users": [
       {
-        "name": "adi",
-        "passwordHash": "$1$.RGu8J4x$U7uxcOg/eotTEIRxhk62I0",
+        "name": "ansibleUser",
         "sshAuthorizedKeys": [
-          "ssh-rsa ..fillyouruser"
+          "ssh-rsa ..publickey.."
         ],
         "groups": [ "wheel" ]
       }
diff --git a/roles/bootstrap-os/defaults/main.yml b/roles/bootstrap-os/defaults/main.yml
index d0b328168..a19f82898 100644
--- a/roles/bootstrap-os/defaults/main.yml
+++ b/roles/bootstrap-os/defaults/main.yml
@@ -19,9 +19,10 @@ fedora_coreos_packages:
   - dbus-tools              # because of networkManager reload bug (https://bugzilla.redhat.com/show_bug.cgi?id=1745659)
   - ethtool                 # required in kubeadm preflight phase for verifying the environment
   - ipset                   # required in kubeadm preflight phase for verifying the environment
+  - conntrack-tools         # required by kube-proxy
 
 ## General
 # Set the hostname to inventory_hostname
 override_system_hostname: true
 
-is_fedora_coreos: false
\ No newline at end of file
+is_fedora_coreos: false
diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml
index 19bbdcff4..dd75ec4fe 100644
--- a/roles/container-engine/cri-o/defaults/main.yml
+++ b/roles/container-engine/cri-o/defaults/main.yml
@@ -1,2 +1,8 @@
 ---
 crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-crio-114-candidate/x86_64/os/'
+
+crio_seccomp_profile: "/etc/crio/seccomp.json"
+
+crio_cgroup_manager: "{{ kubelet_cgroup_driver | default('cgroupfs') }}"
+
+crio_runc_path: "/usr/sbin/runc"
diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml
index e30e94f97..8764b6ec7 100644
--- a/roles/container-engine/cri-o/tasks/main.yaml
+++ b/roles/container-engine/cri-o/tasks/main.yaml
@@ -71,15 +71,33 @@
   register: need_bootstrap_crio
   when: is_ostree
 
+- name: Enable modular repos for crio
+  ini_file:
+    path: "/etc/yum.repos.d/{{ item }}.repo"
+    section: "{{ item }}"
+    option: enabled
+    value: 1
+  become: true
+  when:
+    - is_ostree
+    - not need_bootstrap_crio.stat.exists
+  loop:
+    - "fedora-updates-modular"
+    - "fedora-modular"
+
 - name: Install cri-o packages with osttree
-  raw: "export http_proxy={{ http_proxy | default('') }} && rpm-ostree install {{ crio_packages|join(' ') }}"
-  when: is_ostree and not need_bootstrap_crio.stat.exists
+  command: "rpm-ostree install {{ crio_packages|join(' ') }}"
+  when:
+    - is_ostree
+    - not need_bootstrap_crio.stat.exists
   become: true
 
 - name: Reboot immediately for updated ostree
   reboot:
   become: true
-  when: is_ostree and not need_bootstrap_crio.stat.exists
+  when:
+    - is_ostree
+    - not need_bootstrap_crio.stat.exists
 
 - name: Install cri-o config
   template:
diff --git a/roles/container-engine/cri-o/templates/crio.conf.j2 b/roles/container-engine/cri-o/templates/crio.conf.j2
index f521eefc1..f632f563b 100644
--- a/roles/container-engine/cri-o/templates/crio.conf.j2
+++ b/roles/container-engine/cri-o/templates/crio.conf.j2
@@ -102,20 +102,14 @@ selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }}
 
 # Path to the seccomp.json profile which is used as the default seccomp profile
 # for the runtime.
-{% if ansible_os_family == "ClearLinux" %}
-seccomp_profile = "/usr/share/defaults/crio/seccomp.json"
-{% elif ansible_distribution == "Ubuntu" or is_fedora_coreos %}
-seccomp_profile = ""
-{% else %}
-seccomp_profile = "/etc/crio/seccomp.json"
-{% endif %}
+seccomp_profile = "{{crio_seccomp_profile}}"
 
 # Used to change the name of the default AppArmor profile of CRI-O. The default
 # profile name is "crio-default-" followed by the version string of CRI-O.
 apparmor_profile = "crio-default"
 
 # Cgroup management implementation used for the runtime.
-cgroup_manager = "cgroupfs"
+cgroup_manager = "{{crio_cgroup_manager}}"
 
 # List of default capabilities for containers. If it is empty or commented out,
 # only the capabilities defined in the containers json file by the user/kube
@@ -218,13 +212,7 @@ ctr_stop_timeout = 0
   # of trust of the workload.
 
   [crio.runtime.runtimes.runc]
-{% if ansible_os_family == "ClearLinux" or ansible_os_family == "RedHat" %}
-  runtime_path = "/usr/bin/runc"
-{% elif ansible_distribution == "Ubuntu" %}
-  runtime_path = "/usr/lib/cri-o-runc/sbin/runc"
-{% else %}
-  runtime_path = "/usr/sbin/runc"
-{% endif %}
+  runtime_path = "{{ crio_runc_path }}"
   runtime_type = "oci"
 
 
@@ -293,7 +281,7 @@ network_dir = "/etc/cni/net.d/"
 # Paths to directories where CNI plugin binaries are located.
 plugin_dirs = [
 	"/usr/libexec/cni",
-{% if ansible_os_family == "ClearLinux" %}
+{% if ansible_os_family == "ClearLinux" or is_ostree %}
 	"/opt/cni/bin/",
 {% endif %}
 ]
diff --git a/roles/container-engine/cri-o/vars/clearlinux.yml b/roles/container-engine/cri-o/vars/clearlinux.yml
index bcaed568c..5f258da8e 100644
--- a/roles/container-engine/cri-o/vars/clearlinux.yml
+++ b/roles/container-engine/cri-o/vars/clearlinux.yml
@@ -4,3 +4,5 @@ crio_packages:
 
 crio_service: crio
 crio_conmon: /usr/libexec/crio/conmon
+crio_seccomp_profile: /usr/share/defaults/crio/seccomp.json
+crio_runc_path: /usr/bin/runc
diff --git a/roles/container-engine/cri-o/vars/fedora.yml b/roles/container-engine/cri-o/vars/fedora.yml
index 00dd69ed4..59af9824b 100644
--- a/roles/container-engine/cri-o/vars/fedora.yml
+++ b/roles/container-engine/cri-o/vars/fedora.yml
@@ -5,3 +5,4 @@ crio_packages:
 
 crio_service: cri-o
 crio_conmon: /usr/libexec/crio/conmon
+crio_seccomp_profile: ""
diff --git a/roles/container-engine/cri-o/vars/redhat.yml b/roles/container-engine/cri-o/vars/redhat.yml
index d1924fc58..1d0c62a8f 100644
--- a/roles/container-engine/cri-o/vars/redhat.yml
+++ b/roles/container-engine/cri-o/vars/redhat.yml
@@ -5,3 +5,4 @@ crio_packages:
 
 crio_service: crio
 crio_conmon: /usr/libexec/crio/conmon
+crio_runc_path: /usr/bin/runc
diff --git a/roles/container-engine/cri-o/vars/ubuntu.yml b/roles/container-engine/cri-o/vars/ubuntu.yml
index ba3eaff90..bc8f1f14c 100644
--- a/roles/container-engine/cri-o/vars/ubuntu.yml
+++ b/roles/container-engine/cri-o/vars/ubuntu.yml
@@ -3,4 +3,6 @@ crio_packages:
   - "cri-o-{{ kube_version | regex_replace('^v(?P<major>\\d+).(?P<minor>\\d+).(?P<patch>\\d+)$', '\\g<major>.\\g<minor>') }}"
 
 crio_service: crio
-crio_conmon: /usr/bin/conmon
+crio_conmon: /usr/libexec/podman/conmon
+crio_seccomp_profile: ""
+crio_runc_path: /usr/lib/cri-o-runc/sbin/runc
-- 
GitLab