diff --git a/cluster.yml b/cluster.yml index 8462ea894ae7fa34403ccbda41c7ef7ffc94d1d1..4fc852d973df5e0000987ef56d03fd9e8ce4fe1e 100644 --- a/cluster.yml +++ b/cluster.yml @@ -33,7 +33,8 @@ roles: - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - - { role: docker, tags: docker, when: manage_docker|default(true) } + - { role: docker, tags: docker, when: container_manager == 'docker' } + - { role: cri-o, tags: crio, when: container_manager == 'crio' } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" diff --git a/docs/cri-o.md b/docs/cri-o.md new file mode 100644 index 0000000000000000000000000000000000000000..796b7513a1f71c5dae90da462bd8ac6da9aff458 --- /dev/null +++ b/docs/cri-o.md @@ -0,0 +1,31 @@ +cri-o +=============== + +cri-o is container developed by kubernetes project. +Currently, only basic function is supported for cri-o. + +* cri-o is supported kubernetes 1.11.1 or later. +* helm and other feature may not be supported due to docker dependency. +* scale.yml and upgrade-cluster.yml are not supported. + +helm and other feature may not be supported due to docker dependency. + +Use cri-o instead of docker, set following variable: + +#### all.yml + +``` +kubeadm_enable: true +... +download_container: false +skip_downloads: false +``` + +#### k8s-cluster.yml + +``` +etcd_deployment_type: host +kubelet_deployment_type: host +container_manager: crio +``` + diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index e2829cb986fa53257246aca612bb362270518593..e347f4f1788f9b2cb002652058d98aa80c9cd984 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -155,3 +155,7 @@ bin_dir: /usr/local/bin # Does coreos need auto upgrade, default is true #coreos_auto_upgrade: true + +# Set true to download and cache container +#download_container: true + diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index aa0210ebd34b5ca75b00e144975ca168eb8cb0eb..eb1d01cb9a8b929aedb5a58ab09af79b64567a60 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -135,6 +135,10 @@ skydns_server_secondary: "{{ kube_service_addresses|ipaddr('net')|ipaddr(4)|ipad dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}" dns_domain: "{{ cluster_name }}" +# Container runtime +# docker for docker and crio for cri-o. +container_manager: docker + # Path used to store Docker data docker_daemon_graph: "/var/lib/docker" diff --git a/roles/cri-o/defaults/main.yml b/roles/cri-o/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..3ae39da22ce8afe190202268423ac81274af016f --- /dev/null +++ b/roles/cri-o/defaults/main.yml @@ -0,0 +1,2 @@ +--- +crio_rhel_repo_base_url: 'https://cbs.centos.org/repos/paas7-openshift-origin311-candidate/x86_64/os/' diff --git a/roles/cri-o/files/mounts.conf b/roles/cri-o/files/mounts.conf new file mode 100644 index 0000000000000000000000000000000000000000..b7cde9d8a3bd12cb25f86b28323df58fe867293b --- /dev/null +++ b/roles/cri-o/files/mounts.conf @@ -0,0 +1 @@ +/usr/share/rhel/secrets:/run/secrets diff --git a/roles/cri-o/tasks/main.yaml b/roles/cri-o/tasks/main.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d9e67c866758a295a3cc53087a99cef41637291 --- /dev/null +++ b/roles/cri-o/tasks/main.yaml @@ -0,0 +1,56 @@ +--- +- name: gather os specific variables + include_vars: "{{ item }}" + with_first_found: + - files: + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_release }}.yml" + - "{{ ansible_distribution|lower }}-{{ ansible_distribution_major_version|lower|replace('/', '_') }}.yml" + - "{{ ansible_distribution|lower }}.yml" + - "{{ ansible_os_family|lower }}-{{ ansible_architecture }}.yml" + - "{{ ansible_os_family|lower }}.yml" + - defaults.yml + paths: + - ../vars + skip: true + tags: + - facts + +- name: Add OpenShift Origin repository + yum_repository: + name: origin + description: OpenShift Origin Repo + baseurl: "{{ crio_rhel_repo_base_url }}" + gpgcheck: no + when: ansible_distribution in ["CentOS","RedHat"] and not is_atomic + +- name: Install cri-o packages + package: + name: "{{ item }}" + state: present + with_items: "{{ crio_packages }}" + +- name: Install cri-o config + template: + src: crio.conf.j2 + dest: /etc/crio/crio.conf + +- name: Copy mounts.conf + copy: + src: mounts.conf + dest: /etc/containers/mounts.conf + when: + - ansible_os_family == 'RedHat' + +- name: Create directory for oci hooks + file: + path: /etc/containers/oci/hooks.d + state: directory + owner: root + mode: 0755 + +- name: Install cri-o service + service: + name: "{{ crio_service }}" + enabled: yes + state: restarted diff --git a/roles/cri-o/templates/crio.conf.j2 b/roles/cri-o/templates/crio.conf.j2 new file mode 100644 index 0000000000000000000000000000000000000000..b20a50c707620575794b0f9002dee0c1c176ac98 --- /dev/null +++ b/roles/cri-o/templates/crio.conf.j2 @@ -0,0 +1,234 @@ + +# The "crio" table contains all of the server options. +[crio] + +# CRI-O reads its storage defaults from the containers/storage configuration +# file, /etc/containers/storage.conf. Modify storage.conf if you want to +# change default storage for all tools that use containers/storage. If you +# want to modify just crio, you can change the storage configuration in this +# file. + +# root is a path to the "root directory". CRIO stores all of its data, +# including container images, in this directory. +#root = "/var/lib/containers/storage" + +# run is a path to the "run directory". CRIO stores all of its state +# in this directory. +#runroot = "/var/run/containers/storage" + +# storage_driver select which storage driver is used to manage storage +# of images and containers. +storage_driver = "overlay2" + +# storage_option is used to pass an option to the storage driver. +#storage_option = [ +#] + +# The "crio.api" table contains settings for the kubelet/gRPC interface. +[crio.api] + +# listen is the path to the AF_LOCAL socket on which crio will listen. +listen = "/var/run/crio/crio.sock" + +# stream_address is the IP address on which the stream server will listen +stream_address = "" + +# stream_port is the port on which the stream server will listen +stream_port = "10010" + +# stream_enable_tls enables encrypted tls transport of the stream server +stream_enable_tls = false + +# stream_tls_cert is the x509 certificate file path used to serve the encrypted stream. +# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +stream_tls_cert = "" + +# stream_tls_key is the key file path used to serve the encrypted stream. +# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +stream_tls_key = "" + +# stream_tls_ca is the x509 CA(s) file used to verify and authenticate client +# communication with the tls encrypted stream. +# This file can change, and CRIO will automatically pick up the changes within 5 minutes. +stream_tls_ca = "" + +# file_locking is whether file-based locking will be used instead of +# in-memory locking +file_locking = true + +# The "crio.runtime" table contains settings pertaining to the OCI +# runtime used and options for how to set up and manage the OCI runtime. +[crio.runtime] + +# runtime is the OCI compatible runtime used for trusted container workloads. +# This is a mandatory setting as this runtime will be the default one +# and will also be used for untrusted container workloads if +# runtime_untrusted_workload is not set. +runtime = "/usr/bin/runc" + +# runtime_untrusted_workload is the OCI compatible runtime used for untrusted +# container workloads. This is an optional setting, except if +# default_container_trust is set to "untrusted". +runtime_untrusted_workload = "" + +# default_workload_trust is the default level of trust crio puts in container +# workloads. It can either be "trusted" or "untrusted", and the default +# is "trusted". +# Containers can be run through different container runtimes, depending on +# the trust hints we receive from kubelet: +# - If kubelet tags a container workload as untrusted, crio will try first to +# run it through the untrusted container workload runtime. If it is not set, +# crio will use the trusted runtime. +# - If kubelet does not provide any information about the container workload trust +# level, the selected runtime will depend on the default_container_trust setting. +# If it is set to "untrusted", then all containers except for the host privileged +# ones, will be run by the runtime_untrusted_workload runtime. Host privileged +# containers are by definition trusted and will always use the trusted container +# runtime. If default_container_trust is set to "trusted", crio will use the trusted +# container runtime for all containers. +default_workload_trust = "trusted" + +# no_pivot instructs the runtime to not use pivot_root, but instead use MS_MOVE +no_pivot = false + +# conmon is the path to conmon binary, used for managing the runtime. +conmon = "/usr/libexec/crio/conmon" + +# conmon_env is the environment variable list for conmon process, +# used for passing necessary environment variable to conmon or runtime. +conmon_env = [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +] + +# selinux indicates whether or not SELinux will be used for pod +# separation on the host. If you enable this flag, SELinux must be running +# on the host. +selinux = {{ (preinstall_selinux_state == 'enforcing')|lower }} + +# seccomp_profile is the seccomp json profile path which is used as the +# default for the runtime. +seccomp_profile = "/etc/crio/seccomp.json" + +# apparmor_profile is the apparmor profile name which is used as the +# default for the runtime. +apparmor_profile = "crio-default" + +# cgroup_manager is the cgroup management implementation to be used +# for the runtime. +cgroup_manager = "cgroupfs" + +# default_capabilities is the list of capabilities to add and can be modified here. +# If capabilities below is commented out, the default list of capabilities defined in the +# spec will be added. +# If capabilities is empty below, only the capabilities defined in the container json +# file by the user/kube will be added. +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FSETID", + "FOWNER", + "NET_RAW", + "SETGID", + "SETUID", + "SETPCAP", + "NET_BIND_SERVICE", + "SYS_CHROOT", + "KILL", +] + +# hooks_dir_path is the oci hooks directory for automatically executed hooks +hooks_dir_path = "/usr/share/containers/oci/hooks.d" + +# default_mounts is the mounts list to be mounted for the container when created +# deprecated, will be taken out in future versions, add default mounts to either +# /usr/share/containers/mounts.conf or /etc/containers/mounts.conf +default_mounts = [ +] + +# CRI-O reads its default mounts from the following two files: +# 1) /etc/containers/mounts.conf - this is the override file, where users can +# either add in their own default mounts, or override the default mounts shipped +# with the package. +# 2) /usr/share/containers/mounts.conf - this is the default file read for mounts. +# If you want CRI-O to read from a different, specific mounts file, you can change +# the default_mounts_file path right below. Note, if this is done, CRI-O will only add +# mounts it finds in this file. + +# default_mounts_file is the file path holding the default mounts to be mounted for the +# container when created. +# default_mounts_file = "" + +# pids_limit is the number of processes allowed in a container +pids_limit = 1024 + +# log_size_max is the max limit for the container log size in bytes. +# Negative values indicate that no limit is imposed. +log_size_max = -1 + +# read-only indicates whether all containers will run in read-only mode +read_only = false + +# The "crio.image" table contains settings pertaining to the +# management of OCI images. + +# uid_mappings specifies the UID mappings to have in the user namespace. +# A range is specified in the form containerUID:HostUID:Size. Multiple +# ranges are separed by comma. +uid_mappings = "" + +# gid_mappings specifies the GID mappings to have in the user namespace. +# A range is specified in the form containerGID:HostGID:Size. Multiple +# ranges are separed by comma. +gid_mappings = "" + +[crio.image] + +# default_transport is the prefix we try prepending to an image name if the +# image name as we receive it can't be parsed as a valid source reference +default_transport = "docker://" + +# pause_image is the image which we use to instantiate infra containers. +pause_image = "docker://k8s.gcr.io/pause:3.1" + +# pause_command is the command to run in a pause_image to have a container just +# sit there. If the image contains the necessary information, this value need +# not be specified. +pause_command = "/pause" + +# signature_policy is the name of the file which decides what sort of policy we +# use when deciding whether or not to trust an image that we've pulled. +# Outside of testing situations, it is strongly advised that this be left +# unspecified so that the default system-wide policy will be used. +signature_policy = "" + +# image_volumes controls how image volumes are handled. +# The valid values are mkdir and ignore. +image_volumes = "mkdir" + +# CRI-O reads its configured registries defaults from the containers/image configuration +# file, /etc/containers/registries.conf. Modify registries.conf if you want to +# change default registries for all tools that use containers/image. If you +# want to modify just crio, you can change the registies configuration in this +# file. + +# insecure_registries is used to skip TLS verification when pulling images. +insecure_registries = [ + "{{ kube_service_addresses }}" +] + +# registries is used to specify a comma separated list of registries to be used +# when pulling an unqualified image (e.g. fedora:rawhide). +registries = [ + "docker.io" +] + +# The "crio.network" table contains settings pertaining to the +# management of CNI plugins. +[crio.network] + +# network_dir is is where CNI network configuration +# files are stored. +network_dir = "/etc/cni/net.d/" + +# plugin_dir is is where CNI plugin binaries are stored. +plugin_dir = "/opt/cni/bin/" diff --git a/roles/cri-o/vars/redhat.yml b/roles/cri-o/vars/redhat.yml new file mode 100644 index 0000000000000000000000000000000000000000..962dc9a0a04acfc3fccc460963e7d618b8a0b0cb --- /dev/null +++ b/roles/cri-o/vars/redhat.yml @@ -0,0 +1,7 @@ +--- +crio_packages: + - cri-o + - cri-tools + - oci-systemd-hook + +crio_service: crio \ No newline at end of file diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 97c8d856214e69d5f1581f2c3dd12a06bc823f13..d673c72b554106207dfb68a48b8144914c2a4199 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -10,6 +10,9 @@ skip_downloads: false download_run_once: False download_compress: 1 +# if this is set to true will download container +download_container: True + # if this is set to true, uses the localhost for download_run_once mode # (requires docker and sudo to access docker). You may want this option for # local caching of docker images or for Container Linux by CoreOS cluster nodes. @@ -48,8 +51,12 @@ cilium_version: "v1.2.0" # Download URLs kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/{{ image_arch }}/kubeadm" vault_download_url: "https://releases.hashicorp.com/vault/{{ vault_version }}/vault_{{ vault_version }}_linux_{{ image_arch }}.zip" +etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz" +hyperkube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64/hyperkube" # Checksums +etcd_checksum: b729db0732448064271ea6fdcb901773c4fe917763ca07776f22d0e5e0bd4097 +hyperkube_checksum: d727f8cae3fc26b1add9b4ff0d4d9b99605544ff7fb3baeecdca394362adbfb8 kubeadm_checksum: 6b17720a65b8ff46efe92a5544f149c39a221910d89939838d75581d4e6924c0 vault_binary_checksum: 3c4d70ba71619a43229e65c67830e30e050eab7a81ac6b28325ff707e5914188 @@ -173,6 +180,19 @@ downloads: sha256: "{{ etcd_digest_checksum|default(None) }}" groups: - etcd + etcd_file: + enabled: true + file: true + version: "{{ etcd_version }}" + dest: "etcd-{{ etcd_version }}-linux-amd64.tar.gz" + sha256: "{{ etcd_checksum }}" + source_url: "{{ etcd_download_url }}" + url: "{{ etcd_download_url }}" + unarchive: true + owner: "root" + mode: "0755" + groups: + - etcd kubeadm: enabled: "{{ kubeadm_enabled }}" file: true @@ -194,6 +214,19 @@ downloads: sha256: "{{ hyperkube_digest_checksum|default(None) }}" groups: - k8s-cluster + hyperkube_file: + enabled: true + file: true + version: "{{ kube_version }}" + dest: "hyperkube" + sha256: "{{ hyperkube_checksum }}" + source_url: "{{ hyperkube_download_url }}" + url: "{{ hyperkube_download_url }}" + unarchive: false + owner: "root" + mode: "0755" + groups: + - k8s-cluster cilium: enabled: "{{ kube_network_plugin == 'cilium' }}" container: true diff --git a/roles/download/tasks/download_prep.yml b/roles/download/tasks/download_prep.yml index 1fd7abf2fb29a77a8332a9ba5f78117f1953e85e..b44da45dae714d275349e081b18ec8bea4f6f4a4 100644 --- a/roles/download/tasks/download_prep.yml +++ b/roles/download/tasks/download_prep.yml @@ -7,6 +7,7 @@ failed_when: false changed_when: false check_mode: no + when: download_container - name: container_download | Create dest directory for saved/loaded container images file: @@ -15,6 +16,7 @@ recurse: yes mode: 0755 owner: "{{ansible_ssh_user|default(ansible_user_id)}}" + when: download_container - name: container_download | create local directory for saved/loaded container images file: @@ -28,5 +30,6 @@ when: - download_run_once - download_delegate == 'localhost' + - download_container tags: - localhost diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml index 6a317fd8981b3cf0101510685acdc469086446e3..1984f626d701cd40dfa14834de28df6d0b17e1e4 100644 --- a/roles/download/tasks/main.yml +++ b/roles/download/tasks/main.yml @@ -11,6 +11,7 @@ when: - not skip_downloads|default(false) - item.value.enabled + - (not (item.value.container|default(False))) or (item.value.container and download_container) - name: "Sync container" include_tasks: sync_container.yml diff --git a/roles/etcd/tasks/install_host.yml b/roles/etcd/tasks/install_host.yml index 1d06a7d5ac03951326960912363c9d9091550614..0dc226e666e4af23f05e4f5ed1aec7b4c29f4b0a 100644 --- a/roles/etcd/tasks/install_host.yml +++ b/roles/etcd/tasks/install_host.yml @@ -1,13 +1,21 @@ --- -- name: Install | Copy etcdctl and etcd binary from docker container - command: sh -c "{{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy; - {{ docker_bin_dir }}/docker create --name etcdctl-binarycopy {{ etcd_image_repo }}:{{ etcd_image_tag }} && - {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcdctl {{ bin_dir }}/etcdctl && - {{ docker_bin_dir }}/docker cp etcdctl-binarycopy:/usr/local/bin/etcd {{ bin_dir }}/etcd && - {{ docker_bin_dir }}/docker rm -f etcdctl-binarycopy" - register: etcd_task_result - until: etcd_task_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" +- name: install | Copy etcd binary from download dir + shell: | + rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcd" "{{ bin_dir }}/etcd" + rsync -piu "{{ local_release_dir }}/etcd-{{ etcd_version }}-linux-amd64/etcdctl" "{{ bin_dir }}/etcdctl" changed_when: false when: etcd_cluster_setup + +- name: install | Set etcd binary permissions + file: + path: "{{ bin_dir }}/etcd" + mode: "0755" + state: file + when: etcd_cluster_setup + +- name: install | Set etcdctl binary permissions + file: + path: "{{ bin_dir }}/etcdctl" + mode: "0755" + state: file + when: etcd_cluster_setup \ No newline at end of file diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 index 38ac215a2a6d557886fe714796d822dd9857900b..35ed7a3e6318a4d061f389206983b8f181d155e1 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 @@ -15,3 +15,6 @@ discoveryTokenAPIServers: discoveryTokenUnsafeSkipCAVerification: true nodeRegistration: name: {{ inventory_hostname }} +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% endif %} diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index 69ad06e4f422ca6d96909d34178c25fd2ac999ad..5b33e199e6f082e33a4d1ad1541a44912b119f09 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -97,6 +97,14 @@ kubeadm_config_api_fqdn: "{{ apiserver_loadbalancer_domain_name|default('lb-apiserver.kubernetes.local') }}" when: loadbalancer_apiserver is defined +- name: kubeadm | Copy etcd ca file as k8s ca + command: "cp -T {{ etcd_cert_dir }}/ca.pem {{ kube_config_dir }}/ssl/etcd/ca.crt" + changed_when: false + +- name: kubeadm | Copy etcd cakey as k8s cakey + command: "cp -T {{ etcd_cert_dir }}/ca-key.pem {{ kube_config_dir }}/ssl/etcd/ca.key" + changed_when: false + - name: kubeadm | Create kubeadm config template: src: "kubeadm-config.{{ kubeadmConfig_api_version }}.yaml.j2" @@ -104,7 +112,7 @@ register: kubeadm_config - name: kubeadm | Initialize first master - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all + command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all register: kubeadm_init # Retry is because upload config sometimes fails retries: 3 @@ -114,7 +122,7 @@ - name: kubeadm | Upgrade first master command: >- - timeout -k 240s 240s + timeout -k 600s 600s {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml @@ -167,7 +175,7 @@ when: inventory_hostname != groups['kube-master']|first - name: kubeadm | Init other uninitialized masters - command: timeout -k 240s 240s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all + command: timeout -k 600s 600s {{ bin_dir }}/kubeadm init --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml --ignore-preflight-errors=all register: kubeadm_init when: inventory_hostname != groups['kube-master']|first and not kubeadm_ca.stat.exists failed_when: kubeadm_init.rc != 0 and "field is immutable" not in kubeadm_init.stderr @@ -175,7 +183,7 @@ - name: kubeadm | Upgrade other masters command: >- - timeout -k 240s 240s + timeout -k 600s 600s {{ bin_dir }}/kubeadm upgrade apply -y {{ kube_version }} --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index be2044e312009b35eb1e8d50282a1066ef123c1d..93da9760bccb26717ea747d34866e83992e6e2a9 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -9,27 +9,19 @@ - import_tasks: encrypt-at-rest.yml when: kube_encrypt_secret_data -- name: Compare host kubectl with hyperkube container - command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubectl" - register: kubectl_task_compare_result - until: kubectl_task_compare_result.rc in [0,1,2] - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" +- name: install | Copy kubectl binary from download dir + command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubectl" changed_when: false - failed_when: "kubectl_task_compare_result.rc not in [0,1,2]" tags: - hyperkube - kubectl - upgrade -- name: Copy kubectl from hyperkube container - command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubectl" - when: kubectl_task_compare_result.rc != 0 - register: kubectl_task_result - until: kubectl_task_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" - changed_when: false +- name: install | Set kubectl binary permissions + file: + path: "{{ bin_dir }}/kubectl" + mode: "0755" + state: file tags: - hyperkube - kubectl @@ -37,7 +29,7 @@ - name: Install kubectl bash completion shell: "{{ bin_dir }}/kubectl completion bash >/etc/bash_completion.d/kubectl.sh" - when: kubectl_task_compare_result.rc != 0 and ansible_os_family in ["Debian","RedHat"] + when: ansible_os_family in ["Debian","RedHat"] tags: - kubectl diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index fd569b8876b9c439863ab699f6572a37acc84bcc..f9fb621b530bb7beb01634ce80b962825b4eda57 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -1,4 +1,4 @@ -apiVersion: kubeadm.k8s.io/v1alpha1 +apiVersion: kubeadm.k8s.io/v1alpha2 kind: MasterConfiguration api: advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }} @@ -7,13 +7,14 @@ api: controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }} {% endif %} etcd: - endpoints: + external: + endpoints: {% for endpoint in etcd_access_addresses.split(',') %} - - {{ endpoint }} + - {{ endpoint }} {% endfor %} - caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem - certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem - keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem + caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem + certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem + keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -27,6 +28,12 @@ kubeProxy: {% if kube_proxy_mode == 'ipvs' and kube_version | version_compare('v1.10', '<') %} featureGates: SupportIPVSProxyMode=true mode: ipvs +{% elif kube_proxy_mode == 'ipvs' %} +kubeProxy: + config: + featureGates: + SupportIPVSProxyMode: true + mode: ipvs {% endif %} {% if kube_proxy_nodeport_addresses %} nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}] diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 31c499e0fd3b96ff0a42031e8059b42c4ba930e3..3f123b24db4ab406d7526dfa7c4c3121d645195d 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -138,3 +138,6 @@ nodeRegistration: taints: - effect: NoSchedule key: node-role.kubernetes.io/master +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% endif %} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 2c541c1121f46b4cf8b71319b0521ea1e5f16779..0e73d79320511c02672670a91bd5d0b5423dce62 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -31,6 +31,11 @@ kubelet_cgroups_per_qos: true # Set to empty to avoid cgroup creation kubelet_enforce_node_allocatable: "\"\"" +# Set runtime cgroups +kubelet_runtime_cgroups: "/systemd/system.slice" +# Set kubelet cgroups +kubelet_kubelet_cgroups: "/systemd/system.slice" + # Set false to enable sharing a pid namespace between containers in a pod. # Note that PID namespace sharing requires docker >= 1.13.1. kubelet_disable_shared_pid: true diff --git a/roles/kubernetes/node/tasks/install_host.yml b/roles/kubernetes/node/tasks/install_host.yml index 7fcb4a01d0d5ef3810cd828e29b8313a8a6a18b3..3ca92384805107e335e3882e1d2eeea9be3c01c5 100644 --- a/roles/kubernetes/node/tasks/install_host.yml +++ b/roles/kubernetes/node/tasks/install_host.yml @@ -1,23 +1,17 @@ --- -- name: install | Compare host kubelet with hyperkube container - command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /usr/bin/cmp /hyperkube /systembindir/kubelet" - register: kubelet_task_compare_result - until: kubelet_task_compare_result.rc in [0,1,2] - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" + +- name: install | Copy kubelet binary from download dir + command: rsync -piu "{{ local_release_dir }}/hyperkube" "{{ bin_dir }}/kubelet" changed_when: false - failed_when: "kubelet_task_compare_result.rc not in [0,1,2]" tags: - hyperkube - upgrade -- name: install | Copy kubelet from hyperkube container - command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp -f /hyperkube /systembindir/kubelet" - when: kubelet_task_compare_result.rc != 0 - register: kubelet_task_result - until: kubelet_task_result.rc == 0 - retries: 4 - delay: "{{ retry_stagger | random + 3 }}" +- name: install | Set kubelet binary permissions + file: + path: "{{ bin_dir }}/kubelet" + mode: "0755" + state: file tags: - hyperkube - upgrade diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index aca97ae1221bd80ada115c6a292ec85a1fd73610..7597fd9ae44266ff0ae71b3dd4a7c4cfaa9b3f84 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -34,7 +34,13 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --node-status-update-frequency={{ kubelet_status_update_frequency }} \ --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ --max-pods={{ kubelet_max_pods }} \ +{% if container_manager == 'docker' %} --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ +{% endif %} +{% if container_manager == 'crio' %} +--container-runtime=remote \ +--container-runtime-endpoint=/var/run/crio/crio.sock \ +{% endif %} --anonymous-auth=false \ --read-only-port={{ kube_read_only_port }} \ {% if kube_version | version_compare('v1.8', '<') %} @@ -42,6 +48,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} --fail-swap-on={{ kubelet_fail_swap_on|default(true)}} \ {% endif %} +--runtime-cgroups={{ kubelet_runtime_cgroups }} --kubelet-cgroups={{ kubelet_kubelet_cgroups }} \ {% endset %} {# Node reserved CPU/memory #} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index c99194ba9a08a7841c12a6ad6b3841589cf5dd9f..ae4654424be1f2a31c068831396a3042301c03e3 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -15,7 +15,9 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" --cadvisor-port={{ kube_cadvisor_port }} \ --pod-infra-container-image={{ pod_infra_image_repo }}:{{ pod_infra_image_tag }} \ --node-status-update-frequency={{ kubelet_status_update_frequency }} \ +{% if container_manager == 'docker' %} --docker-disable-shared-pid={{ kubelet_disable_shared_pid }} \ +{% endif %} --client-ca-file={{ kube_cert_dir }}/ca.pem \ --tls-cert-file={{ kube_cert_dir }}/node-{{ inventory_hostname }}.pem \ --tls-private-key-file={{ kube_cert_dir }}/node-{{ inventory_hostname }}-key.pem \ @@ -26,6 +28,10 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% if kube_version | version_compare('v1.7', '<') %} --enable-cri={{ kubelet_enable_cri }} \ {% endif %} +{% if container_manager == 'crio' %} +--container-runtime=remote \ +--container-runtime-endpoint=/var/run/crio/crio.sock \ +{% endif %} --cgroup-driver={{ kubelet_cgroup_driver|default(kubelet_cgroup_driver_detected) }} \ --cgroups-per-qos={{ kubelet_cgroups_per_qos }} \ --max-pods={{ kubelet_max_pods }} \ diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 626c797bc86b4280005d72553d883c798010a625..54986fe253401f56c004165cc065ac7c48c85bca 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -131,9 +131,8 @@ kube_apiserver_insecure_port: 8080 # Aggregator kube_api_aggregator_routing: false -# Docker options -# Optionally do not run docker role -manage_docker: true +# Container for runtime +container_manager: docker # Path used to store Docker data docker_daemon_graph: "/var/lib/docker" @@ -366,3 +365,6 @@ etcd_events_peer_addresses: |- {%- endfor %} podsecuritypolicy_enabled: false +etcd_heartbeat_interval: "250" +etcd_election_timeout: "5000" +etcd_snapshot_count: "10000" diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index 47b51546f32911e64a1f50f4728768ec8189fa14..88dec8d7a5ae74e2a7b1e97208ad94cf83dc1bfc 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -60,6 +60,16 @@ tags: - docker +- name: reset | remove all cri-o containers + shell: "crictl ps -aq | xargs -r crictl rm" + register: remove_all_crio_containers + retries: 4 + until: remove_all_crio_containers.rc == 0 + delay: 5 + tags: + - crio + when: container_manager == 'crio' + - name: reset | gather mounted kubelet dirs shell: mount | grep /var/lib/kubelet/ | awk '{print $3}' | tac check_mode: no diff --git a/scale.yml b/scale.yml index 676fba610ceed00adf5ff95cd03fd1bf62b9f0cf..c4cd117f00cd07cd3ffcccda8787c9cb7554b14f 100644 --- a/scale.yml +++ b/scale.yml @@ -35,7 +35,9 @@ roles: - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - - { role: docker, tags: docker, when: manage_docker|default(true) } + + - { role: docker, tags: docker, when: container_manager == 'docker' } + - { role: cri-o, tags: crio, when: container_manager == 'crio' } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]" diff --git a/upgrade-cluster.yml b/upgrade-cluster.yml index abc89e18fce6f614b572b451a0694f2dd93556fb..7d8534d782c82a7e2d64ef0613b13c9f7b003ae0 100644 --- a/upgrade-cluster.yml +++ b/upgrade-cluster.yml @@ -34,7 +34,8 @@ roles: - { role: kubespray-defaults} - { role: kubernetes/preinstall, tags: preinstall } - - { role: docker, tags: docker, when: manage_docker|default(true) } + - { role: docker, tags: docker, when: container_manager == 'docker' } + - { role: cri-o, tags: crio, when: container_manager == 'crio' } - role: rkt tags: rkt when: "'rkt' in [etcd_deployment_type, kubelet_deployment_type, vault_deployment_type]"