README.md

@@ -163,11 +163,11 @@ Note: Upstart/SysV init based OS types are not supported.
   - [kubernetes](https://github.com/kubernetes/kubernetes) v1.29.3
   - [etcd](https://github.com/etcd-io/etcd) v3.5.12
   - [docker](https://www.docker.com/) v24.0 (see [Note](#container-runtime-notes))
-  - [containerd](https://containerd.io/) v1.7.15
+  - [containerd](https://containerd.io/) v1.7.16
   - [cri-o](http://cri-o.io/) v1.29.1 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) v1.2.0
-  - [calico](https://github.com/projectcalico/calico) v3.27.2
+  - [calico](https://github.com/projectcalico/calico) v3.27.3
   - [cilium](https://github.com/cilium/cilium) v1.15.4
   - [flannel](https://github.com/flannel-io/flannel) v0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) v1.11.5

requirements.txt

 ansible==9.3.0
 cryptography==41.0.4
-jinja2==3.1.2
+jinja2==3.1.3
 jmespath==1.0.1
 MarkupSafe==2.1.3
 netaddr==0.9.0

roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml

@@ -19,5 +19,6 @@ ingress_nginx_without_class: true
 ingress_nginx_default: false
 ingress_nginx_webhook_enabled: false
 ingress_nginx_webhook_job_ttl: 1800
+ingress_nginx_opentelemetry_enabled: false
 
 ingress_nginx_probe_initial_delay_seconds: 10

roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2

@@ -23,6 +23,26 @@ spec:
     spec:
       serviceAccountName: ingress-nginx
       terminationGracePeriodSeconds: {{ ingress_nginx_termination_grace_period_seconds }}
+{% if ingress_nginx_opentelemetry_enabled %}
+      initContainers:
+      - name: opentelemetry
+        command:
+        - /init_module
+        image: {{ ingress_nginx_opentelemetry_image_repo }}:{{ ingress_nginx_opentelemetry_image_tag }}
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 65532
+          allowPrivilegeEscalation: false
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+              - ALL
+          readOnlyRootFilesystem: true
+        volumeMounts:
+        - mountPath: /modules_mount
+          name: modules
+{% endif %}
 {% if ingress_nginx_host_network %}
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
@@ -127,15 +147,27 @@ spec:
             timeoutSeconds: 5
             successThreshold: 1
             failureThreshold: 3
-{% if ingress_nginx_webhook_enabled %}
+{% if ingress_nginx_webhook_enabled or ingress_nginx_opentelemetry_enabled %}
           volumeMounts:
+{% if ingress_nginx_webhook_enabled %}
             - mountPath: /usr/local/certificates/
               name: webhook-cert
               readOnly: true
 {% endif %}
-{% if ingress_nginx_webhook_enabled %}
+{% if ingress_nginx_opentelemetry_enabled %}
+            - name: modules
+              mountPath: /modules_mount
+{% endif %}
+{% endif %}
+{% if ingress_nginx_webhook_enabled or ingress_nginx_opentelemetry_enabled %}
       volumes:
+{% if ingress_nginx_webhook_enabled %}
         - name: webhook-cert
           secret:
             secretName: ingress-nginx-admission
 {% endif %}
+{% if ingress_nginx_opentelemetry_enabled %}
+        - name: modules
+          emptyDir: {}
+{% endif %}
+{% endif %}

roles/kubernetes/preinstall/defaults/main.yml

@@ -6,18 +6,6 @@ epel_enabled: false
 # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf
 dns_late: false
 
-common_required_pkgs:
-  - "{{ (ansible_distribution == 'openSUSE Tumbleweed') | ternary('openssl-1_1', 'openssl') }}"
-  - curl
-  - rsync
-  - socat
-  - unzip
-  - e2fsprogs
-  - xfsprogs
-  - ebtables
-  - bash-completion
-  - tar
-
 # Set to true if your network does not support IPv6
 # This may be necessary for pulling Docker images from
 # GCE docker repository

roles/kubernetes/preinstall/files/pkgs-schema.json

+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://kubespray.io/internal/os_packages.schema.json",
+    "title": "Os packages",
+    "description": "Criteria for selecting packages to install on Kubernetes nodes during installation by Kubespray",
+    "type": "object",
+    "patternProperties": {
+        ".*": {
+            "type": "object",
+            "additionalProperties": false,
+            "properties": {
+                "enabled": {
+                    "description": "Escape hatch to filter packages. The value is expected to be pre-resolved to a boolean by Jinja",
+                    "type": "boolean",
+                    "default": true
+                },
+                "groups": {
+                    "description": "Match if the host is in one of these groups. If not specified match any host.",
+                    "type": "array",
+                    "minItems": 1,
+                    "items":{
+                        "type": "string",
+                        "pattern": "^[0-9A-Za-z_]*$"
+                    }
+                },
+                "os": {
+                    "type": "object",
+                    "description": "If not specified match any OS. Otherwise, must match by 'families' or 'distributions' to be included.",
+                    "additionalProperties": false,
+                    "minProperties": 1,
+                    "properties": {
+                        "families": {
+                            "description": "Match if ansible_os_family is part of the list.",
+                            "type": "array",
+                            "minItems": 1,
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "distributions": {
+                            "type": "object",
+                            "description": "Match if ansible_distribution match one of defined keys.",
+                            "minProperties": 1,
+                            "patternProperties": {
+                                ".*": {
+                                    "description": "Match if either the value is the empty hash, or one major_versions/versions/releases contains the corresponding variable ('ansible_distrbution_*')",
+                                    "type": "object",
+                                    "additionalProperties": false,
+                                    "properties": {
+                                        "major_versions": {
+                                            "type": "array",
+                                            "minItems": 1,
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        },
+                                        "versions": {
+                                            "type": "array",
+                                            "minItems": 1,
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        },
+                                        "releases": {
+                                            "type": "array",
+                                            "minItems": 1,
+                                            "items": {
+                                                "type": "string"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

roles/kubernetes/preinstall/tasks/0020-set_facts.yml

@@ -199,20 +199,6 @@
       supersede domain-name-servers {{ (nameservers | d([]) + cloud_resolver | d([])) | unique | join(', ') }};
   when: dns_early and not dns_late
 
-- name: Gather os specific variables
-  include_vars: "{{ item }}"
-  with_first_found:
-    - files:
-        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower | replace('/', '_') }}.yml"
-        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_release }}.yml"
-        - "{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower | replace('/', '_') }}.yml"
-        - "{{ ansible_distribution | lower }}.yml"
-        - "{{ ansible_os_family | lower }}.yml"
-        - defaults.yml
-      paths:
-        - ../vars
-      skip: true
-
 - name: Set etcd vars if using kubeadm mode
   set_fact:
     etcd_cert_dir: "{{ kube_cert_dir }}"

roles/kubernetes/preinstall/tasks/0040-verify-settings.yml

@@ -316,3 +316,15 @@
   when:
     - kube_apiserver_enable_admission_plugins is defined
     - kube_apiserver_enable_admission_plugins | length > 0
+
+- name: Verify that the packages list structure is valid
+  ansible.utils.validate:
+    criteria: "{{ lookup('file', 'pkgs-schema.json') }}"
+    data: "{{ pkgs }}"
+
+- name: Verify that the packages list is sorted
+  vars:
+    pkgs_lists: "{{ pkgs.keys() | list }}"
+  assert:
+    that: "pkgs_lists | sort == pkgs_lists"
+    fail_msg: "pkgs is not sorted: {{ pkgs_lists | ansible.utils.fact_diff(pkgs_lists | sort) }}"

roles/kubernetes/preinstall/tasks/0070-system-packages.yml

@@ -59,19 +59,28 @@
   tags:
     - bootstrap-os
 
-- name: Update common_required_pkgs with ipvsadm when kube_proxy_mode is ipvs
-  set_fact:
-    common_required_pkgs: "{{ common_required_pkgs | default([]) + ['ipvsadm', 'ipset'] }}"
-  when: kube_proxy_mode == 'ipvs'
-
 - name: Install packages requirements
+  vars:
+    # The json_query for selecting packages name is split for readability
+    # see files/pkgs-schema.json for the structure of `pkgs`
+    # and the matching semantics
+    full_query: "[? value | (enabled == null || enabled) && ( {{ filters_os }} ) && ( {{ filters_groups }} ) ].key"
+    filters_groups: "groups | @ == null || [? contains(`{{ group_names }}`, @)]"
+    filters_os: "os == null || (os | ( {{ filters_family }} ) || ( {{ filters_distro }} ))"
+    dquote: !unsafe '"'
+    # necessary to workaround Ansible escaping
+    filters_distro: "distributions.{{ dquote }}{{ ansible_distribution  }}{{ dquote }} |
+                          @ == `{}` ||
+                          contains(not_null(major_versions, `[]`), '{{ ansible_distribution_major_version }}') ||
+                          contains(not_null(versions, `[]`), '{{ ansible_distribution_version }}') ||
+                          contains(not_null(releases, `[]`), '{{ ansible_distribution_release }}')"
+    filters_family: "families && contains(families, '{{ ansible_os_family }}')"
   package:
-    name: "{{ required_pkgs | default([]) | union(common_required_pkgs | default([])) }}"
+    name: "{{ pkgs | dict2items | to_json|from_json | community.general.json_query(full_query) }}"
     state: present
   register: pkgs_task_result
   until: pkgs_task_result is succeeded
   retries: "{{ pkg_install_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
-  when: not (ansible_os_family in ["Flatcar", "Flatcar Container Linux by Kinvolk"] or is_fedora_coreos)
   tags:
     - bootstrap-os

roles/kubernetes/preinstall/vars/amazon.yml

----
-required_pkgs:
-  - libselinux-python
-  - device-mapper-libs
-  - nss
-  - conntrack-tools
-  - libseccomp

roles/kubernetes/preinstall/vars/centos.yml

----
-required_pkgs:
-  - "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
-  - device-mapper-libs
-  - nss
-  - conntrack
-  - container-selinux
-  - libseccomp

roles/kubernetes/preinstall/vars/debian-11.yml

----
-required_pkgs:
-  - python3-apt
-  - gnupg
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - iptables
-  - apparmor
-  - libseccomp2

roles/kubernetes/preinstall/vars/debian-12.yml

----
-required_pkgs:
-  - python3-apt
-  - gnupg
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - iptables
-  - apparmor
-  - libseccomp2
-  - mergerfs

roles/kubernetes/preinstall/vars/debian.yml

----
-required_pkgs:
-  - python-apt
-  - aufs-tools
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - apparmor
-  - libseccomp2

roles/kubernetes/preinstall/vars/fedora.yml

----
-required_pkgs:
-  - iptables
-  - libselinux-python3
-  - device-mapper-libs
-  - conntrack
-  - container-selinux
-  - libseccomp

roles/kubernetes/preinstall/vars/main.yml

+---
+pkgs:
+  apparmor: &debian_family_base
+    os:
+      families:
+      - Debian
+  apt-transport-https: *debian_family_base
+  aufs-tools: &deb_10
+    groups:
+    - k8s_cluster
+    os:
+      distributions:
+        Debian:
+          major_versions:
+          - "10"
+  bash-completion: {}
+  conntrack: &deb_redhat
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Debian
+      - RedHat
+  conntrack-tools:
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Suse
+      distributions:
+        Amazon: {}
+  container-selinux: &redhat_family
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - RedHat
+  curl: {}
+  device-mapper:
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Suse
+  device-mapper-libs: *redhat_family
+  e2fsprogs: {}
+  ebtables: {}
+  gnupg: &debian
+    groups:
+    - k8s_cluster
+    os:
+      distributions:
+        Debian:
+          major_versions:
+          - "11"
+          - "12"
+  ipset:
+    enabled: "{{ kube_proxy_mode != 'ipvs' }}"
+    groups:
+    - k8s_cluster
+  iptables: *deb_redhat
+  ipvsadm:
+    enabled: "{{ kube_proxy_mode == 'ipvs' }}"
+    groups:
+    - k8s_cluster
+  libseccomp: *redhat_family
+  libseccomp2:
+    groups:
+    - k8s_cluster
+    os:
+      families:
+      - Suse
+      - Debian
+  libselinux-python:  # TODO: Handle rehat_family + major < 8
+    os:
+      distributions:
+        Amazon: {}
+  libselinux-python3:
+    os:
+      distributions:
+        Fedora: {}
+  mergerfs:
+    os:
+      distributions:
+        Debian:
+          major_versions:
+          - "12"
+  nss: *redhat_family
+  openssl: {}
+  python-apt: *deb_10
+  # TODO: not for debian 10
+  python3-apt: *debian_family_base
+  python3-libselinux:
+    os:
+      distributions:
+        RedHat: &major_redhat_like
+          major_versions:
+          - "8"
+          - "9"
+        Centos: *major_redhat_like
+  rsync: {}
+  socat: {}
+  software-properties-common: *debian_family_base
+  tar: {}
+  unzip: {}
+  xfsprogs: {}

roles/kubernetes/preinstall/vars/redhat.yml

----
-required_pkgs:
-  - "{{ ((ansible_distribution_major_version | int) < 8) | ternary('libselinux-python', 'python3-libselinux') }}"
-  - device-mapper-libs
-  - nss
-  - conntrack
-  - container-selinux
-  - libseccomp

roles/kubernetes/preinstall/vars/suse.yml

----
-required_pkgs:
-  - device-mapper
-  - conntrack-tools
-  - libseccomp2

roles/kubernetes/preinstall/vars/ubuntu.yml

----
-required_pkgs:
-  - python3-apt
-  - apt-transport-https
-  - software-properties-common
-  - conntrack
-  - apparmor
-  - libseccomp2

roles/kubespray-defaults/defaults/main/checksums.yml

@@ -474,6 +474,7 @@ cni_binary_checksums:
     v1.0.0: 1a055924b1b859c54a97dc14894ecaa9b81d6d949530b9544f0af4173f5a8f2a
 calicoctl_binary_checksums:
   arm:
+    v3.27.3: 0
     v3.27.2: 0
     v3.27.1: 0
     v3.27.0: 0
@@ -495,6 +496,7 @@ calicoctl_binary_checksums:
     v3.23.5: 0
     v3.23.4: 0
   arm64:
+    v3.27.3: 1fc5f58a18d8b1c487b4663fc5cbe23b45bd9d31617debd309f6dfac7c11a8ef
     v3.27.2: 0fd1f65a511338cf9940835987d420c94ab95b5386288ba9673b736a4d347463
     v3.27.1: 0
     v3.27.0: b4b8c71f9658165e45336b9b5e4fad865529feeffe4294247eb5b4c4310dcaf9
@@ -516,6 +518,7 @@ calicoctl_binary_checksums:
     v3.23.5: 0941ad0deeb03d8fda96340948cdbc15d14062086438150cf3ec5ee2767b22c3
     v3.23.4: c54b7d122d9315bbab1a88707b7168a0934a80c4f2a94c9e871bcc8a8cf11c11
   amd64:
+    v3.27.3: e22b8bb41684f8ffb5143b50bf3b2ab76985604d774d397cfb6fb11d8a19f326
     v3.27.2: 692f69dc656e41cd35e23e24f56c98c4aeeb723fed129985b46f71e6eb5e1594
     v3.27.1: 0
     v3.27.0: 46e79ae146b3dd90998f56511cf5d6db64deb97cb784235caf1f99e0672d66e4
@@ -537,6 +540,7 @@ calicoctl_binary_checksums:
     v3.23.5: 4c777881709ddaabcf4b56dcbe683125d7ed5743c036fee9273c5295e522082f
     v3.23.4: 1ea0d3b6543645612e8239978878b6adefdb7619a16ecbdb8e6dc2687538f689
   ppc64le:
+    v3.27.3: 5f2ac510c0ec31ec4c02ff2660f2502b68b655616d5b766a51bd99d2e3604fbc
     v3.27.2: f918bb88de1d01de3d143e1e75d0ee1256f247c5cbabec7d665aaf8d1fd3cc6c
     v3.27.1: 0
     v3.27.0: 3de46d8bc30c6f9d9387d484ed62a5655c1f204b1b831b5a90f0a0d1c1ffd752
@@ -603,6 +607,7 @@ ciliumcli_binary_checksums:
     v0.15.16: 0
     v0.15.15: 0
 calico_crds_archive_checksums:
+  v3.27.3: d11a32919bff389f642af5df8180ad3cec586030decd35adb2a7d4a8aa3b298e
   v3.27.2: 8154bb4aad887f2a5500b505fe203a918f72c4e602b04c688c4b94f76a26e925
   v3.27.1: 76abb0db222af279e3514cfae02be9259097b565bbb2ffcb776ca00566480edb
   v3.27.0: 2a4b5132035dfd6ac4abc8d545f33de139350eca523e0c5cfe4ac32e43fcb2f1
@@ -962,6 +967,7 @@ nerdctl_archive_checksums:
     1.6.1: 3924467d9430df991ebdf4e78211bac2b29e9a066d5000d98f8d4ebde2bb7b4c
 containerd_archive_checksums:
   arm:
+    1.7.16: 0
     1.7.15: 0
     1.7.14: 0
     1.7.13: 0
@@ -978,6 +984,8 @@ containerd_archive_checksums:
     1.7.2: 0
     1.7.1: 0
     1.7.0: 0
+    1.6.31: 0
+    1.6.30: 0
     1.6.29: 0
     1.6.28: 0
     1.6.27: 0
@@ -995,6 +1003,7 @@ containerd_archive_checksums:
     1.6.15: 0
     1.6.14: 0
   arm64:
+    1.7.16: 2d4373de40a6f58cd0f29377c0257b35697a987248e6268520586996771d7a75
     1.7.15: 5cc8bd8f3d9803ef0ef701596e89d62ad6850a2544e722842f4533642df36d87
     1.7.14: 44df66d0a0332465e7d15e90b974cd4f08d059dfa26652218ed9485390f47f9e
     1.7.13: 118759e398f35337109592b4d237538872dc12a207d38832b9d04515d0acbc4d
@@ -1011,6 +1020,8 @@ containerd_archive_checksums:
     1.7.2: d75a4ca53d9addd0b2c50172d168b12957e18b2d8b802db2658f2767f15889a6
     1.7.1: 1f828dc063e3c24b0840b284c5635b5a11b1197d564c97f9e873b220bab2b41b
     1.7.0: e7e5be2d9c92e076f1e2e15c9f0a6e0609ddb75f7616999b843cba92d01e4da2
+    1.6.31: 91a74cc602c7724668537f754006692114af70cfb6ef840b288f922fa68f7ed7
+    1.6.30: 0bbf1eed508d6ebc240b900648c76f12a07c0c6125aa8c22d46c9ce24252f9e3
     1.6.29: 0
     1.6.28: 96a231f875ddf9cc7682b881d408ae993f2bd5d0a40402a74ec4fda672047427
     1.6.27: 433b0e8113adfd726374e04fc2f61dafad65c53db5665569f2715a7a916a1813
@@ -1028,6 +1039,7 @@ containerd_archive_checksums:
     1.6.15: d63e4d27c51e33cd10f8b5621c559f09ece8a65fec66d80551b36cac9e61a07d
     1.6.14: 3ccb61218e60cbba0e1bbe1e5e2bf809ac1ead8eafbbff36c3195d3edd0e4809
   amd64:
+    1.7.16: 4f4f2c3c7d14fd59a404961a3a3341303c2fdeeba0e78808c209f606e828f99c
     1.7.15: ea27e6454954bd9cb62a70b0a40eb085ae9c96cb8c075a74910102b33586e07d
     1.7.14: 48e0d9747cd51cb90e0b278d100397653d9f2e765effca194427e4796395b240
     1.7.13: c2371c009dd8b7738663333d91e5ab50d204f8bcae24201f45d59060d12c3a23
@@ -1044,6 +1056,8 @@ containerd_archive_checksums:
     1.7.2: 2755c70152ab40856510b4549c2dd530e15f5355eb7bf82868e813c9380e22a7
     1.7.1: 9504771bcb816d3b27fab37a6cf76928ee5e95a31eb41510a7d10ae726e01e85
     1.7.0: b068b05d58025dc9f2fc336674cac0e377a478930f29b48e068f97c783a423f0
+    1.6.31: 52080601f414b7e63a5b8e0cb8c1d641c9e070447ac96da9b1aeb00480744ba5
+    1.6.30: 1f1b65190b626883394e6f2ecbe5141afc6c45fc1ca035ef052e66bb2c479a5f
     1.6.29: 0
     1.6.28: b2f15c722d1cc8b74ed643068e043b92bd031fc23d53488d1e837cf4b2777391
     1.6.27: 8c0b04a8b39127c084d490cca905d565c94929dd15e168b0f8663076fdcf5539
@@ -1061,6 +1075,7 @@ containerd_archive_checksums:
     1.6.15: 191bb4f6e4afc237efc5c85b5866b6fdfed731bde12cceaa6017a9c7f8aeda02
     1.6.14: 7da626d46c4edcae1eefe6d48dc6521db3e594a402715afcddc6ac9e67e1bfcd
   ppc64le:
+    1.7.16: d0add7a55a5d4411cafb276469d2b78bc3ada11cb4b444b9e35f9ef60c00960d
     1.7.15: b38641d9bd18139495cf9839999039b19941f53d36a6d72efe4577c489dfda0c
     1.7.14: b84b523909b9dd0c0b2bc40bd2b9af543ec9f1186df69c220ae3749e34623dbb
     1.7.13: 89605ed2365d5eb779477d11947101236eb44e5244f1e58bb162a9e68d242798
@@ -1077,6 +1092,8 @@ containerd_archive_checksums:
     1.7.2: cbe7ec913cb603ca218bd8867efdce4bee3b0e0115e467e51c910467daf8184e
     1.7.1: 17d97ef55c6ce7af9778dbafb5e73f577d1b34220043a91cccde49dbcc610342
     1.7.0: 051e897d3ee5b8c8097f65be447fea2d29226b583ca5d9ed78e9aebcf4e69889
+    1.6.31: 4458a2398f27241b6e674ea9ba1f56dc4d9ab9dacc5a07469602776c3e428110
+    1.6.30: ba3d790f504a845b060e2faae3cc0603afd125ebdddb3bdb513b8d70a4337d87
     1.6.29: 0
     1.6.28: 35411f9d1bafc9cae91c2e30d46a59d01bffc7e18ac7f0942dca9d1d5907ab38
     1.6.27: 8106915bc62c51383baa77925748505e79229fded7efcd7e74cb352ad10d0ce9