From 785366c2de50851b03749fc0eb5f33e3c4d63fe9 Mon Sep 17 00:00:00 2001
From: Takuya Murakami <murakami_da@nec.com>
Date: Wed, 31 Jan 2024 17:57:23 +0900
Subject: [PATCH] [kubernetes] Support kubernetes 1.29 (#10820)
* [kubernetes] Make kubernetes 1.29.1 default
* [cri-o]: support cri-o 1.29
Use "crio status" instead of "crio-status" for cri-o >=1.29.0
* Remove GAed feature gates SecCompDefault
The SecCompDefault feature gate was removed since k8s 1.29
https://github.com/kubernetes/kubernetes/pull/121246
---
README.md | 4 ++--
docs/hardening.md | 2 +-
inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml | 2 +-
roles/container-engine/cri-o/defaults/main.yml | 7 -------
roles/container-engine/cri-o/tasks/load_vars.yml | 8 ++++++++
roles/container-engine/cri-o/tasks/main.yaml | 5 ++++-
roles/container-engine/cri-o/tasks/reset.yml | 3 +++
roles/container-engine/cri-o/vars/v1.28.yml | 9 +++++++++
roles/container-engine/cri-o/vars/v1.29.yml | 9 +++++++++
roles/kubernetes/node/tasks/facts.yml | 2 +-
roles/kubespray-defaults/defaults/main/download.yml | 4 ++--
roles/kubespray-defaults/defaults/main/main.yml | 2 +-
.../packet_ubuntu20-calico-all-in-one-hardening.yml | 2 +-
13 files changed, 42 insertions(+), 17 deletions(-)
create mode 100644 roles/container-engine/cri-o/tasks/load_vars.yml
create mode 100644 roles/container-engine/cri-o/vars/v1.28.yml
create mode 100644 roles/container-engine/cri-o/vars/v1.29.yml
diff --git a/README.md b/README.md
index 2565680a7..897e58505 100644
--- a/README.md
+++ b/README.md
@@ -160,11 +160,11 @@ Note: Upstart/SysV init based OS types are not supported.
## Supported Components
- Core
- - [kubernetes](https://github.com/kubernetes/kubernetes) v1.28.6
+ - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.1
- [etcd](https://github.com/etcd-io/etcd) v3.5.10
- [docker](https://www.docker.com/) v20.10 (see note)
- [containerd](https://containerd.io/) v1.7.11
- - [cri-o](http://cri-o.io/) v1.27 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
+ - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
- Network Plugin
- [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0
- [calico](https://github.com/projectcalico/calico) v3.26.4
diff --git a/docs/hardening.md b/docs/hardening.md
index fe2f3a568..8623bdc30 100644
--- a/docs/hardening.md
+++ b/docs/hardening.md
@@ -97,7 +97,7 @@ kubelet_event_record_qps: 1
kubelet_rotate_certificates: true
kubelet_streaming_connection_idle_timeout: "5m"
kubelet_make_iptables_util_chains: true
-kubelet_feature_gates: ["RotateKubeletServerCertificate=true", "SeccompDefault=true"]
+kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
kubelet_seccomp_default: true
kubelet_systemd_hardening: true
# In case you have multiple interfaces in your
diff --git a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
index 68233b4c8..37ef5e2ca 100644
--- a/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml
@@ -17,7 +17,7 @@ kube_token_dir: "{{ kube_config_dir }}/tokens"
kube_api_anonymous_auth: true
## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.28.6
+kube_version: v1.29.1
# Where the binaries will be downloaded.
# Note: ensure that you've enough disk space (about 1G)
diff --git a/roles/container-engine/cri-o/defaults/main.yml b/roles/container-engine/cri-o/defaults/main.yml
index f20108f86..6bc0d7238 100644
--- a/roles/container-engine/cri-o/defaults/main.yml
+++ b/roles/container-engine/cri-o/defaults/main.yml
@@ -78,13 +78,6 @@ crio_subuid_length: 16777216
crio_subgid_start: 2130706432
crio_subgid_length: 16777216
-# cri-o binary files
-crio_bin_files:
- - conmon
- - crio
- - crio-status
- - pinns
-
# cri-o manual files
crio_man_files:
5:
diff --git a/roles/container-engine/cri-o/tasks/load_vars.yml b/roles/container-engine/cri-o/tasks/load_vars.yml
new file mode 100644
index 000000000..8a5daa392
--- /dev/null
+++ b/roles/container-engine/cri-o/tasks/load_vars.yml
@@ -0,0 +1,8 @@
+---
+- name: Cri-o | include vars/v1.28.yml
+ include_vars: v1.28.yml
+ when: crio_version is version("v1.29.0", operator="<")
+
+- name: Cri-o | include vars/v1.29.yml
+ include_vars: v1.29.yml
+ when: crio_version is version("v1.29.0", operator=">=")
diff --git a/roles/container-engine/cri-o/tasks/main.yaml b/roles/container-engine/cri-o/tasks/main.yaml
index 60c818a55..99a3ed278 100644
--- a/roles/container-engine/cri-o/tasks/main.yaml
+++ b/roles/container-engine/cri-o/tasks/main.yaml
@@ -1,4 +1,7 @@
---
+- name: Cri-o | load vars
+ import_tasks: load_vars.yml
+
- name: Cri-o | check if fedora coreos
stat:
path: /run/ostree-booted
@@ -203,7 +206,7 @@
- not service_start.changed
- name: Cri-o | verify that crio is running
- command: "{{ bin_dir }}/crio-status info"
+ command: "{{ bin_dir }}/{{ crio_status_command }} info"
register: get_crio_info
until: get_crio_info is succeeded
changed_when: false
diff --git a/roles/container-engine/cri-o/tasks/reset.yml b/roles/container-engine/cri-o/tasks/reset.yml
index 1bae013ac..53d471434 100644
--- a/roles/container-engine/cri-o/tasks/reset.yml
+++ b/roles/container-engine/cri-o/tasks/reset.yml
@@ -1,4 +1,7 @@
---
+- name: Cri-o | load vars
+ import_tasks: load_vars.yml
+
- name: CRI-O | Kubic repo name for debian os family
set_fact:
crio_kubic_debian_repo_name: "{{ ((ansible_distribution == 'Ubuntu') | ternary('x', '')) ~ ansible_distribution ~ '_' ~ ansible_distribution_version }}"
diff --git a/roles/container-engine/cri-o/vars/v1.28.yml b/roles/container-engine/cri-o/vars/v1.28.yml
new file mode 100644
index 000000000..a6a8b673d
--- /dev/null
+++ b/roles/container-engine/cri-o/vars/v1.28.yml
@@ -0,0 +1,9 @@
+---
+# cri-o binary files
+crio_bin_files:
+ - conmon
+ - crio
+ - crio-status
+ - pinns
+
+crio_status_command: crio-status
diff --git a/roles/container-engine/cri-o/vars/v1.29.yml b/roles/container-engine/cri-o/vars/v1.29.yml
new file mode 100644
index 000000000..bc25e2e56
--- /dev/null
+++ b/roles/container-engine/cri-o/vars/v1.29.yml
@@ -0,0 +1,9 @@
+---
+# cri-o binary files
+crio_bin_files:
+ - conmon
+ - conmonrs
+ - crio
+ - pinns
+
+crio_status_command: crio status
diff --git a/roles/kubernetes/node/tasks/facts.yml b/roles/kubernetes/node/tasks/facts.yml
index 156d748d2..0aaa11d60 100644
--- a/roles/kubernetes/node/tasks/facts.yml
+++ b/roles/kubernetes/node/tasks/facts.yml
@@ -18,7 +18,7 @@
when: container_manager == 'crio'
block:
- name: Look up crio cgroup driver
- shell: "set -o pipefail && {{ bin_dir }}/crio-status info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
+ shell: "set -o pipefail && {{ bin_dir }}/{{ crio_status_command }} info | grep 'cgroup driver' | awk -F': ' '{ print $2; }'"
args:
executable: /bin/bash
register: crio_cgroup_driver_result
diff --git a/roles/kubespray-defaults/defaults/main/download.yml b/roles/kubespray-defaults/defaults/main/download.yml
index cc8d676f7..000954c30 100644
--- a/roles/kubespray-defaults/defaults/main/download.yml
+++ b/roles/kubespray-defaults/defaults/main/download.yml
@@ -133,9 +133,9 @@ skopeo_version: "v1.13.2"
kube_major_version: "{{ kube_version | regex_replace('^v([0-9])+\\.([0-9]+)\\.[0-9]+', 'v\\1.\\2') }}"
pod_infra_supported_versions:
+ v1.29: "3.9"
v1.28: "3.9"
v1.27: "3.9"
- v1.26: "3.9"
pod_infra_version: "{{ pod_infra_supported_versions[kube_major_version] }}"
etcd_supported_versions:
@@ -158,9 +158,9 @@ crio_version: "{{ crio_supported_versions[kube_major_version] }}"
# Scheduler plugins doesn't build for K8s 1.28 yet
scheduler_plugins_supported_versions:
+ v1.29: 0
v1.28: 0
v1.27: v0.27.8
- v1.26: v0.26.7
scheduler_plugins_version: "{{ scheduler_plugins_supported_versions[kube_major_version] }}"
yq_version: "v4.35.2"
diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml
index f7eb5b373..114668ac5 100644
--- a/roles/kubespray-defaults/defaults/main/main.yml
+++ b/roles/kubespray-defaults/defaults/main/main.yml
@@ -16,7 +16,7 @@ kubelet_fail_swap_on: true
kubelet_swap_behavior: LimitedSwap
## Change this to use another Kubernetes version, e.g. a current beta release
-kube_version: v1.28.6
+kube_version: v1.29.1
## The minimum version working
kube_version_min_required: v1.27.0
diff --git a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml
index d8dcc1f8e..55cbd5063 100644
--- a/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml
+++ b/tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml
@@ -86,7 +86,7 @@ kubelet_event_record_qps: 1
kubelet_rotate_certificates: true
kubelet_streaming_connection_idle_timeout: "5m"
kubelet_make_iptables_util_chains: true
-kubelet_feature_gates: ["RotateKubeletServerCertificate=true", "SeccompDefault=true"]
+kubelet_feature_gates: ["RotateKubeletServerCertificate=true"]
kubelet_seccomp_default: true
kubelet_systemd_hardening: true
# In case you have multiple interfaces in your
--
GitLab