kubeadm support (#1631)

* kubeadm support

* move k8s master to a subtask
* disable k8s secrets when using kubeadm
* fix etcd cert serial var
* move simple auth users to master role
* make a kubeadm-specific env file for kubelet
* add non-ha CI job

* change ci boolean vars to json format

* fixup

* Update create-gce.yml

* Update create-gce.yml

* Update create-gce.yml

.gitlab-ci.yml

@@ -53,6 +53,7 @@ before_script:
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   UPGRADE_TEST: "false"
+  KUBEADM_ENABLED: "false"
   RESOLVCONF_MODE: docker_dns
   LOG_LEVEL: "-vv"
   ETCD_DEPLOYMENT: "docker"
@@ -117,9 +118,9 @@ before_script:
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cert_management=${CERT_MGMT:-script}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
@@ -127,6 +128,9 @@ before_script:
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
       -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml
@@ -144,17 +148,19 @@ before_script:
       -e ansible_ssh_user=${SSH_USER}
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e cloud_provider=gce
-      -e deploy_netchecker=true
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kube_network_plugin=${KUBE_NETWORK_PLUGIN}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
       -e local_release_dir=${PWD}/downloads
       -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
       -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
       -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       $PLAYBOOK;
@@ -178,14 +184,18 @@ before_script:
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e deploy_netchecker=true
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -221,14 +231,18 @@ before_script:
       --private-key=${HOME}/.ssh/id_rsa
       -e bootstrap_os=${BOOTSTRAP_OS}
       -e ansible_python_interpreter=${PYPATH}
-      -e download_localhost=${DOWNLOAD_LOCALHOST}
-      -e download_run_once=${DOWNLOAD_RUN_ONCE}
-      -e deploy_netchecker=true
-      -e resolvconf_mode=${RESOLVCONF_MODE}
-      -e local_release_dir=${PWD}/downloads
+      -e "{deploy_netchecker: true}"
+      -e "{download_localhost: ${DOWNLOAD_LOCALHOST}}"
+      -e "{download_run_once: ${DOWNLOAD_RUN_ONCE}}"
       -e etcd_deployment_type=${ETCD_DEPLOYMENT}
       -e kubedns_min_replicas=1
       -e kubelet_deployment_type=${KUBELET_DEPLOYMENT}
+      -e local_release_dir=${PWD}/downloads
+      -e resolvconf_mode=${RESOLVCONF_MODE}
+      -e vault_deployment_type=${VAULT_DEPLOYMENT}
+      -e "{kubeadm_enabled: ${KUBEADM_ENABLED}}"
+      -e weave_cpu_requests=${WEAVE_CPU_LIMIT}
+      -e weave_cpu_limit=${WEAVE_CPU_LIMIT}
       -e "${AUTHORIZATION_MODES}"
       --limit "all:!fake_hosts"
       cluster.yml;
@@ -280,6 +294,17 @@ before_script:
   UPGRADE_TEST: "graceful"
   STARTUP_SCRIPT: ""
 
+.ubuntu_canal_kubeadm_variables: &ubuntu_canal_kubeadm_variables
+# stage: deploy-gce-part1
+  KUBE_NETWORK_PLUGIN: canal
+  AUTHORIZATION_MODES: "{ 'authorization_modes':  [ 'RBAC' ] }"
+  CLOUD_IMAGE: ubuntu-1604-xenial
+  CLOUD_MACHINE_TYPE: "n1-standard-2"
+  CLOUD_REGION: europe-west1-b
+  CLUSTER_MODE: default
+  KUBEADM_ENABLED: "true"
+  STARTUP_SCRIPT: ""
+
 .rhel7_weave_variables: &rhel7_weave_variables
 # stage: deploy-gce-part1
   KUBE_NETWORK_PLUGIN: weave
@@ -470,6 +495,27 @@ ubuntu-canal-ha-rbac-triggers:
   when: on_success
   only: ['triggers']
 
+ubuntu-canal-kubeadm-rbac:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: manual
+  except: ['triggers']
+  only: ['master', /^pr-.*$/]
+
+ubuntu-canal-kubeadm-triggers:
+  stage: deploy-gce-part1
+  <<: *job
+  <<: *gce
+  variables:
+    <<: *gce_variables
+    <<: *ubuntu_canal_kubeadm_variables
+  when: on_success
+  only: ['triggers']
+
 rhel7-weave:
   stage: deploy-gce-part1
   <<: *job

cluster.yml

@@ -69,6 +69,17 @@
   roles:
     - { role: kubespray-defaults}
     - { role: kubernetes/master, tags: master }
+
+- hosts: k8s-cluster
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
+    - { role: kubernetes/kubeadm, tags: kubeadm, when: "kubeadm_enabled" }
+
+- hosts: kube-master
+  any_errors_fatal: "{{ any_errors_fatal | default(true) }}"
+  roles:
+    - { role: kubespray-defaults}
     - { role: kubernetes-apps/network_plugin, tags: network }
     - { role: kubernetes-apps/policy_controller, tags: policy-controller }
 

inventory/group_vars/all.yml

@@ -82,6 +82,12 @@ bin_dir: /usr/local/bin
 #openstack_lbaas_monitor_timeout: "30s"
 #openstack_lbaas_monitor_max_retries: "3"
 
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+#kubeadm_token_first: "{{ lookup('password', 'credentials/kubeadm_token_first length=6  chars=ascii_letters,digits') }}"
+#kubeadm_token_second: "{{ lookup('password', 'credentials/kubeadm_token_second length=16 chars=ascii_letters,digits') }}"
+#kubeadm_token: "{{ kubeadm_token_first }}.{{ kubeadm_token_second }}"
+#
 ## Set these proxy values in order to update docker daemon to use proxies
 #http_proxy: ""
 #https_proxy: ""

library/kube.py

@@ -135,12 +135,15 @@ class KubeManager(object):
             return None
         return out.splitlines()
 
-    def create(self, check=True):
+    def create(self, check=True, force=True):
         if check and self.exists():
             return []
 
         cmd = ['apply']
 
+        if force:
+            cmd.append('--force')
+
         if not self.filename:
             self.module.fail_json(msg='filename required to create')
 
@@ -148,11 +151,11 @@ class KubeManager(object):
 
         return self._execute(cmd)
 
-    def replace(self):
+    def replace(self, force=True):
 
         cmd = ['apply']
 
-        if self.force:
+        if force:
             cmd.append('--force')
 
         if not self.filename:

roles/download/defaults/main.yml

@@ -19,6 +19,7 @@ download_always_pull: False
 
 # Versions
 kube_version: v1.7.3
+kubeadm_version: "{{ kube_version }}"
 etcd_version: v3.2.4
 # TODO(mattymo): Move calico versions to roles/network_plugins/calico/defaults
 # after migration to container download
@@ -31,11 +32,13 @@ flannel_version: "v0.8.0"
 flannel_cni_version: "v0.2.0"
 pod_infra_version: 3.0
 
-# Download URL's
+# Download URLs
 etcd_download_url: "https://storage.googleapis.com/kargo/{{etcd_version}}_etcd"
+kubeadm_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kubeadm_version }}/bin/linux/amd64/kubeadm"
 
 # Checksums
 etcd_checksum: "274c46a7f8d26f7ae99d6880610f54933cbcf7f3beafa19236c52eb5df8c7a0b"
+kubeadm_checksum: "378e6052f8b178f8e6a38e8637681c72d389443b66b78b51b8ddc9a162c655c3"
 
 # Containers
 # Possible values: host, docker
@@ -132,6 +135,15 @@ downloads:
     container: "{{ etcd_deployment_type in [ 'docker', 'rkt' ] }}"
     repo: "{{ etcd_image_repo }}"
     tag: "{{ etcd_image_tag }}"
+  kubeadm:
+    version: "{{ kubeadm_version }}"
+    dest: "kubeadm"
+    sha256: "{{ kubeadm_checksum }}"
+    source_url: "{{ kubeadm_download_url }}"
+    url: "{{ kubeadm_download_url }}"
+    unarchive: false
+    owner: "root"
+    mode: "0755"
   hyperkube:
     container: true
     repo: "{{ hyperkube_image_repo }}"

roles/etcd/tasks/main.yml

@@ -11,7 +11,7 @@
 
 - name: "Gen_certs | Get etcd certificate serials"
   shell: "openssl x509 -in {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem -noout -serial | cut -d= -f2"
-  register: "node-{{ inventory_hostname }}_serial"
+  register: "etcd_client_cert_serial"
   when: inventory_hostname in groups['k8s-cluster']|union(groups['etcd'])|union(groups['calico-rr']|default([]))|unique|sort
 
 - include: "install_{{ etcd_deployment_type }}.yml"

roles/kubernetes-apps/ansible/tasks/main.yml

@@ -8,6 +8,18 @@
   delay: 6
   when: inventory_hostname == groups['kube-master'][0]
 
+- name: kubeadm | Delete kubeadm kubedns
+  kube:
+    name: "kubedns"
+    namespace: "{{ system_namespace }}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "deploy"
+    state: absent
+  when:
+    - kubeadm_enabled|default(false)
+    - kubeadm_init.changed|default(false)
+    - inventory_hostname == groups['kube-master'][0]
+
 - name: Kubernetes Apps | Lay Down KubeDNS Template
   template:
     src: "{{item.file}}"

roles/kubernetes/kubeadm/tasks/main.yml

+---
+- name: Set kubeadm_discovery_address
+  set_fact:
+    kubeadm_discovery_address: >-
+      {%- if "127.0.0.1" or "localhost" in kube_apiserver_endpoint -%}
+      {{ first_kube_master }}:{{ kube_apiserver_port }}
+      {%- else -%}
+      {{ kube_apiserver_endpoint }}
+      {%- endif %}
+  when: not is_kube_master
+  tags: facts
+
+- name: Create kubeadm client config
+  template:
+    src: kubeadm-client.conf.j2
+    dest: "{{ kube_config_dir }}/kubeadm-client.conf"
+    backup: yes
+  when: not is_kube_master
+  register: kubeadm_client_conf
+
+- name: Join to cluster if needed
+  command: kubeadm join --config {{ kube_config_dir}}/kubeadm-client.conf --skip-preflight-checks
+  register: kubeadm_join
+  when: not is_kube_master and kubeadm_client_conf.changed
+
+- name: Update server field in kubelet kubeconfig
+  replace:
+    path: "{{ kube_config_dir }}/kubelet.conf"
+    regexp: '(\s+){{ first_kube_master }}:{{ kube_apiserver_port }}(\s+.*)?$'
+    replace: '\1{{ kube_apiserver_endpoint }}\2'
+    backup: yes
+  when: not is_kube_master and kubeadm_discovery_address != kube_apiserver_endpoint
+
+# FIXME(mattymo): Reconcile kubelet kubeconfig filename for both deploy modes
+- name: Symlink kubelet kubeconfig for calico/canal
+  file:
+    src: "{{ kube_config_dir }}//kubelet.conf"
+    dest: "{{ kube_config_dir }}/node-kubeconfig.yaml"
+    state: link
+    force: yes
+  when: kube_network_plugin in ['calico','canal']

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.j2

+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: NodeConfiguration
+caCertPath: {{ kube_config_dir }}/ssl/ca.crt
+token: {{ kubeadm_token }}
+discoveryTokenAPIServers:
+- {{ kubeadm_discovery_address | replace("https://", "")}}

roles/kubernetes/master/defaults/main.yml

@@ -66,3 +66,7 @@ apiserver_custom_flags: []
 controller_mgr_custom_flags: []
 
 scheduler_custom_flags: []
+
+# kubeadm settings
+# Value of 0 means it never expires
+kubeadm_token_ttl: 0

roles/kubernetes/master/handlers/main.yml

@@ -44,3 +44,7 @@
   until: result.status == 200
   retries: 20
   delay: 6
+
+- name: Master | set secret_changed
+  set_fact:
+    secret_changed: true

roles/kubernetes/master/tasks/kubeadm-setup.yml

+---
+- name: kubeadm | aggregate all SANs
+  set_fact:
+    apiserver_sans: >-
+      kubernetes
+      kubernetes.default
+      kubernetes.default.svc
+      kubernetes.default.svc.{{ dns_domain }}
+      {{ kube_apiserver_ip }}
+      localhost
+      127.0.0.1
+      {{ ' '.join(groups['kube-master']) }}
+      {%- if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %}
+      {{ apiserver_loadbalancer_domain_name }}
+      {%- endif %}
+      {%- for host in groups['kube-master'] -%}
+      {%- if hostvars[host]['access_ip'] is defined %}{{ hostvars[host]['access_ip'] }}{% endif -%}
+      {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+      {%- endfor %}
+  tags: facts
+
+- name: kubeadm | Copy etcd cert dir under k8s cert dir
+  command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
+  changed_when: false
+
+- name: kubeadm | Create kubeadm config
+  template:
+    src: kubeadm-config.yaml.j2
+    dest: "{{ kube_config_dir }}/kubeadm-config.yaml"
+  register: kubeadm_config
+
+- name: kubeadm | Initialize cluster
+  command: timeout -k 240s 240s kubeadm init --config={{ kube_config_dir }}/kubeadm-config.yaml --skip-preflight-checks
+  register: kubeadm_init
+  when: kubeadm_config.changed

roles/kubernetes/master/tasks/main.yml

@@ -2,6 +2,9 @@
 - include: pre-upgrade.yml
   tags: k8s-pre-upgrade
 
+- include: users-file.yml
+  when: kube_basic_auth|default(true)
+
 - name: Copy kubectl from hyperkube container
   command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl"
   register: kube_task_result
@@ -25,63 +28,10 @@
   when: ansible_os_family in ["Debian","RedHat"]
   tags: [kubectl, upgrade]
 
-- name: Write kube-apiserver manifest
-  template:
-    src: manifests/kube-apiserver.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
-  notify: Master | wait for the apiserver to be running
-  tags: kube-apiserver
-
-- meta: flush_handlers
-
-- name: Write kube system namespace manifest
-  template:
-    src: namespace.j2
-    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  run_once: yes
-  when: inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Check if kube system namespace exists
-  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
-  register: 'kubesystem'
-  changed_when: False
-  failed_when: False
-  run_once: yes
-  tags: apps
-
-- name: Create kube system namespace
-  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
-  retries: 4
-  delay: "{{ retry_stagger | random + 3 }}"
-  register: create_system_ns
-  until: create_system_ns.rc == 0
-  changed_when: False
-  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
-  tags: apps
-
-- name: Write kube-scheduler kubeconfig
-  template:
-    src: kube-scheduler-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
-  tags: kube-scheduler
-
-- name: Write kube-scheduler manifest
-  template:
-    src: manifests/kube-scheduler.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
-  notify: Master | wait for kube-scheduler
-  tags: kube-scheduler
-
-- name: Write kube-controller-manager kubeconfig
-  template:
-    src: kube-controller-manager-kubeconfig.yaml.j2
-    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
-  tags: kube-controller-manager
+- task: Include kubeadm setup if enabled
+  include: kubeadm-setup.yml
+  when: kubeadm_enabled|bool|default(false)
 
-- name: Write kube-controller-manager manifest
-  template:
-    src: manifests/kube-controller-manager.manifest.j2
-    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
-  notify: Master | wait for kube-controller-manager
-  tags: kube-controller-manager
+- task: Include static pod setup if not using kubeadm
+  include: static-pod-setup.yml
+  when: not kubeadm_enabled|bool|default(false)

roles/kubernetes/master/tasks/static-pod-setup.yml

+---
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apiserver.manifest"
+  notify: Master | wait for the apiserver to be running
+  tags: kube-apiserver
+
+- meta: flush_handlers
+
+- name: Write kube system namespace manifest
+  template:
+    src: namespace.j2
+    dest: "{{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  run_once: yes
+  when: inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Check if kube system namespace exists
+  command: "{{ bin_dir }}/kubectl get ns {{system_namespace}}"
+  register: 'kubesystem'
+  changed_when: False
+  failed_when: False
+  run_once: yes
+  tags: apps
+
+- name: Create kube system namespace
+  command: "{{ bin_dir }}/kubectl create -f {{kube_config_dir}}/{{system_namespace}}-ns.yml"
+  retries: 4
+  delay: "{{ retry_stagger | random + 3 }}"
+  register: create_system_ns
+  until: create_system_ns.rc == 0
+  changed_when: False
+  when: kubesystem|failed and inventory_hostname == groups['kube-master'][0]
+  tags: apps
+
+- name: Write kube-scheduler kubeconfig
+  template:
+    src: kube-scheduler-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler-kubeconfig.yaml"
+  tags: kube-scheduler
+
+- name: Write kube-scheduler manifest
+  template:
+    src: manifests/kube-scheduler.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-scheduler.manifest"
+  notify: Master | wait for kube-scheduler
+  tags: kube-scheduler
+
+- name: Write kube-controller-manager kubeconfig
+  template:
+    src: kube-controller-manager-kubeconfig.yaml.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager-kubeconfig.yaml"
+  tags: kube-controller-manager
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-controller-manager.manifest"
+  notify: Master | wait for kube-controller-manager
+  tags: kube-controller-manager

roles/kubernetes/master/tasks/users-file.yml

+---
+- name: Make sure the users directory exits
+  file:
+    path: "{{ kube_users_dir }}"
+    state: directory
+    mode: o-rwx
+    group: "{{ kube_cert_group }}"
+
+- name: Populate users for basic auth in API
+  template:
+    src: known_users.csv.j2
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    backup: yes
+  notify: Master | set secret_changed

roles/kubernetes/master/templates/kubeadm-config.yaml.j2

+apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+api:
+  advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }}
+  bindPort: "{{ kube_apiserver_port }}"
+etcd:
+  endpoints: 
+{% for endpoint in etcd_access_endpoint.split(',') %}
+  - {{ endpoint }}
+{% endfor %}
+  caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
+  certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
+  keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+networking:
+  dnsDomain: {{ dns_domain }}
+  serviceSubnet: {{ kube_service_addresses }}
+  podSubnet: {{ kube_pods_subnet }}
+kubernetesVersion: {{ kube_version }}
+cloudProvider: {{ cloud_provider|default('') }}
+#TODO: cloud provider conf file
+authorizationModes: 
+{% for mode in authorization_modes %}
+- {{ mode }}
+{% endfor %}
+token: {{ kubeadm_token }}
+tokenTTL: {{ kubeadm_token_ttl }}
+selfHosted: false
+apiServerExtraArgs:
+  insecure-bind-address: {{ kube_apiserver_insecure_bind_address }}
+  insecure-port: "{{ kube_apiserver_insecure_port }}"
+  admission-control: {{ kube_apiserver_admission_control | join(',') }}
+  service-node-port-range: {{ kube_apiserver_node_port_range }}
+{% if kube_basic_auth|default(true) %}
+  basic-auth-file: {{ kube_users_dir }}/known_users.csv
+{% endif %}
+{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %}
+  oidc-issuer-url: {{ kube_oidc_url }}
+  oidc-client-id: {{ kube_oidc_client_id }}
+{%   if kube_oidc_ca_file is defined %}
+  oidc-ca-file: {{ kube_oidc_ca_file }}
+{%   endif %}
+{%   if kube_oidc_username_claim is defined %}
+  oidc-username-claim: {{ kube_oidc_username_claim }}
+{%   endif %}
+{%   if kube_oidc_groups_claim is defined %}
+  oidc-groups-claim: {{ kube_oidc_groups_claim }}
+{%   endif %}
+{% endif %}
+  storage-backend: {{ kube_apiserver_storage_backend }}
+{% if kube_api_runtime_config is defined %}
+  runtime-config: {{ kube_api_runtime_config }}
+{% endif %}
+  allow-privileged: "true"
+#TODO: Custom flags compatible with kubeadm
+controllerManagerExtraArgs:
+  node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
+  node-monitor-period: {{ kube_controller_node_monitor_period }}
+  pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kube_feature_gates %}
+  feature-gates: {{ kube_feature_gates|join(',') }}
+{% endif %}
+#schedulerExtraArgs:
+apiServerCertSANs:
+{% for san in  apiserver_sans.split(' ') | unique %}
+  - {{ san }}
+{% endfor %}
+certificatesDir: {{ kube_config_dir }}/ssl

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -7,7 +7,7 @@ metadata:
     k8s-app: kube-apiserver
     kubespray: v2
   annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     kubespray.apiserver-cert/serial: "{{ apiserver_cert_serial }}"
 spec:
   hostNetwork: true

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

@@ -6,7 +6,7 @@ metadata:
   labels:
     k8s-app: kube-controller
   annotations:
-    kubespray.etcd-cert/serial: "{{ etcd_node_cert_serial }}"
+    kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}"
     kubespray.controller-manager-cert/serial: "{{ controller_manager_cert_serial }}"
 spec:
   hostNetwork: true

roles/kubernetes/node/meta/main.yml

@@ -10,7 +10,12 @@ dependencies:
     file: "{{ downloads.install_socat }}"
     tags: [download, kubelet]
     when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+  - role: download
+    file: "{{ downloads.kubeadm }}"
+    tags: [download, kubelet, kubeadm]
+    when: kubeadm_enabled
   - role: kubernetes/secrets
+    when: not kubeadm_enabled
     tags: k8s-secrets
   - role: download
     file: "{{ downloads.nginx }}"