diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 845f28ca179e265e1d4c67d8d5c353d72ccedb0e..21abfb62977aac010c3571a97deb4e42e2cb4acc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -41,7 +41,7 @@ before_script: tags: - kubernetes - docker - image: quay.io/kubespray/kubespray:v2.7 + image: quay.io/kubespray/kubespray:v2.8 .docker_service: &docker_service services: @@ -88,11 +88,11 @@ before_script: - echo ${PWD} - echo "${STARTUP_SCRIPT}" - cd tests && make create-${CI_PLATFORM} -s ; cd - - #- git fetch --all && git checkout v2.7.0 # Check out latest tag if testing upgrade # Uncomment when gitlab kubespray repo has tags - - test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) + #- test "${UPGRADE_TEST}" != "false" && git fetch --all && git checkout $(git describe --tags $(git rev-list --tags --max-count=1)) + - test "${UPGRADE_TEST}" != "false" && git checkout 9051aa5296ef76fcff69a2e3827cef28752aa475 # Checkout the CI vars file so it is available - test "${UPGRADE_TEST}" != "false" && git checkout "${CI_BUILD_REF}" tests/files/${CI_JOB_NAME}.yml # Workaround https://github.com/kubernetes-sigs/kubespray/issues/2021 diff --git a/README.md b/README.md index 1138419b269d88a6f60391e2bb2bc5565fce933e..67d98bd58745866d8cbadd486aa8dc75e1cab506 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ Supported Components -------------------- - Core - - [kubernetes](https://github.com/kubernetes/kubernetes) v1.12.3 + - [kubernetes](https://github.com/kubernetes/kubernetes) v1.13.0 - [etcd](https://github.com/coreos/etcd) v3.2.24 - [docker](https://www.docker.com/) v18.06 (see note) - [rkt](https://github.com/rkt/rkt) v1.21.0 (see Note 2) diff --git a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml index ffd158859118e785b1dc02a9b382984e54fece1c..e1a8003273dca6fec1f5924ba701fb346a926cbb 100644 --- a/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster/k8s-cluster.yml @@ -19,7 +19,7 @@ kube_users_dir: "{{ kube_config_dir }}/users" kube_api_anonymous_auth: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.12.3 +kube_version: v1.13.0 # kubernetes image repo define kube_image_repo: "gcr.io/google-containers" diff --git a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 index ec7e43fdb673963c7040f851abb19e00dad96264..1b81f4c486c9eea359b0dbb2f4ff54aa21c84598 100644 --- a/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 +++ b/roles/dnsmasq/templates/dnsmasq-autoscaler.yml.j2 @@ -28,7 +28,6 @@ spec: labels: k8s-app: dnsmasq-autoscaler annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]' spec: {% if kube_version is version('v1.11.1', '>=') %} diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 3bc9e0134f6497b7fb90db003f3682f19a3c4434..8dd6ae576a39b32aaf9018814da6cbbe557e1d9d 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -35,7 +35,7 @@ download_delegate: "{% if download_localhost %}localhost{% else %}{{groups['kube image_arch: "{{host_architecture | default('amd64')}}" # Versions -kube_version: v1.12.3 +kube_version: v1.13.0 kubeadm_version: "{{ kube_version }}" etcd_version: v3.2.24 @@ -70,6 +70,7 @@ cni_download_url: "https://github.com/containernetworking/plugins/releases/downl # Checksums hyperkube_checksums: + v1.13.0: 754f1baae5dc2ba29afc66e1f5d3b676ee59cd5c40ccce813092408d53bde3d9 v1.12.3: 600aad3f0d016716abd85931239806193ffbe95f2edfdcea11532d518ae5cdb1 v1.12.2: 566dfed398c20c9944f8999d6370cb584cb8c228b3c5881137b6b3d9306e4b06 v1.12.1: 4aa23cfb2fc2e2e4d0cbe0d83a648c38e4baabd6c66f5cdbbb40cbc7582fdc74 @@ -88,6 +89,7 @@ hyperkube_checksums: v1.10.1: 6e0642ad6bae68dc81b8d1c9efa18e265e17e23da1895862823cafac08c0344c v1.10.0: b5575b2fb4266754c1675b8cd5d9b6cac70f3fee7a05c4e80da3a9e83e58c57e kubeadm_checksums: + v1.13.0: f5366206416dc4cfc840a7add2289957b56ccc479cc1b74f7397a4df995d6b06 v1.12.3: c675aa3be82754b3f8dfdde2a1526a72986713312d46d898e65cb564c6aa8ad4 v1.12.2: 51bc4bfd1d934a27245111c0ad1f793d5147ed15389415a1509502f23fcfa642 v1.12.1: 5d95efd65aad398d85a9802799f36410ae7a95f9cbe73c8b10d2213c10a6d7be diff --git a/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 b/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 index 2d930c6e310e4caad82367fe07b1752b7ebb5a5e..20c86550d75cee3d0b9fbf8160097d9e521ec37e 100644 --- a/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/dns-autoscaler.yml.j2 @@ -31,7 +31,6 @@ spec: labels: k8s-app: dns-autoscaler{{ coredns_ordinal_suffix | default('') }} annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: {% if kube_version is version('v1.11.1', '>=') %} diff --git a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 index fa2507115012c8a80726a18e616e8e99a0ebd584..a095f162529c8858b1693cbfe0db16c7c0c90c78 100644 --- a/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/kubedns-deploy.yml.j2 @@ -25,7 +25,6 @@ spec: labels: k8s-app: kube-dns annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: {% if kube_version is version('v1.11.1', '>=') %} diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 index 84f440442ee452b2dd792787df79843df7976729..5fd84012c87672fae332ea7d49a306c20674c05f 100644 --- a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/k8s-device-plugin-nvidia-daemonset.yml.j2 @@ -14,8 +14,6 @@ spec: metadata: labels: k8s-app: nvidia-gpu-device-plugin - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical affinity: diff --git a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 index a1adede5a29ec9d7df5f71dad927994caa0130bf..5dda821893130cc36da1c6800861aed8c597cd52 100644 --- a/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 +++ b/roles/kubernetes-apps/container_engine_accelerator/nvidia_gpu/templates/nvidia-driver-install-daemonset.yml.j2 @@ -22,8 +22,6 @@ spec: metadata: labels: name: nvidia-driver-installer - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: priorityClassName: system-node-critical affinity: diff --git a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 index 6cb51d025f6292cd85b8eed8f980d0adb969c8bb..245c3beab283855048179147e0e0d77c426a4a5c 100644 --- a/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 +++ b/roles/kubernetes-apps/metrics_server/templates/metrics-server-deployment.yaml.j2 @@ -21,7 +21,6 @@ spec: app.kubernetes.io/name: metrics-server version: {{ metrics_server_version }} annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' seccomp.security.alpha.kubernetes.io/pod: 'docker/default' spec: {% if kube_version is version('v1.11.1', '>=') %} diff --git a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 index dcb7c9f5f3ee6f7685b8617c56723761976cb01e..1eb8e1d4d3760c18d547d2741dee97c4b569be5b 100644 --- a/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 +++ b/roles/kubernetes-apps/policy_controller/calico/templates/calico-kube-controllers.yml.j2 @@ -6,8 +6,6 @@ metadata: labels: k8s-app: calico-kube-controllers kubernetes.io/cluster-service: "true" - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: replicas: 1 strategy: diff --git a/roles/kubernetes/kubeadm/tasks/main.yml b/roles/kubernetes/kubeadm/tasks/main.yml index 29b866a9766640db329889f2b64b5cae8311eb8b..c2035859d377629acc180a81b68e82b4b2f93232 100644 --- a/roles/kubernetes/kubeadm/tasks/main.yml +++ b/roles/kubernetes/kubeadm/tasks/main.yml @@ -46,7 +46,14 @@ - name: sets kubeadm api version to v1alpha3 set_fact: kubeadmConfig_api_version: v1alpha3 - when: kubeadm_output.stdout is version('v1.12.0', '>=') + when: + - kubeadm_output.stdout is version('v1.12.0', '>=') + - kubeadm_output.stdout is version('v1.13.0', '<') + +- name: sets kubeadm api version to v1beta1 + set_fact: + kubeadmConfig_api_version: v1beta1 + when: kubeadm_output.stdout is version('v1.13.0', '>=') - name: Create kubeadm client config template: diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 index fe9f45b2ff21b56f1731096c6f9a68825112053f..6a40ab03e80c1b30ac9fe7445afce2e1fabc9eb3 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha1.j2 @@ -1,6 +1,6 @@ apiVersion: kubeadm.k8s.io/v1alpha1 kind: NodeConfiguration -caCertPath: {{ kube_config_dir }}/ssl/ca.crt +caCertPath: {{ kube_cert_dir }}/ca.crt token: {{ kubeadm_token }} discoveryTokenAPIServers: {% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 index eebcdf7c029d105bf663e7878307bdc3ac481292..b5d8365d76d5e2d567592a58fe2262d7909f1c67 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha2.j2 @@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha2 kind: NodeConfiguration clusterName: {{ cluster_name }} discoveryFile: "" -caCertPath: {{ kube_config_dir }}/ssl/ca.crt +caCertPath: {{ kube_cert_dir }}/ca.crt discoveryTimeout: {{ discovery_timeout }} discoveryToken: {{ kubeadm_token }} tlsBootstrapToken: {{ kubeadm_token }} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 index a1e0887e0aeee45dff8efe09eb9dacf9e21f22ed..ff5f2c0b4f67adb36e0c822069286360aae9c3db 100644 --- a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1alpha3.j2 @@ -2,7 +2,7 @@ apiVersion: kubeadm.k8s.io/v1alpha3 kind: JoinConfiguration clusterName: {{ cluster_name }} discoveryFile: "" -caCertPath: {{ kube_config_dir }}/ssl/ca.crt +caCertPath: {{ kube_cert_dir }}/ca.crt discoveryTimeout: {{ discovery_timeout }} discoveryToken: {{ kubeadm_token }} tlsBootstrapToken: {{ kubeadm_token }} diff --git a/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 new file mode 100644 index 0000000000000000000000000000000000000000..2eb0cf368c1ae0e62273307ea2028ca2ac185454 --- /dev/null +++ b/roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta1.j2 @@ -0,0 +1,27 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: JoinConfiguration +discovery: + bootstrapToken: +{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %} + apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} + apiServerEndpoint: {{ kubeadm_discovery_address | replace("https://", "")}} +{% endif %} + token: {{ kubeadm_token }} + unsafeSkipCAVerification: true + timeout: {{ discovery_timeout }} + tlsBootstrapToken: {{ kubeadm_token }} +{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %} +controlPlane: + localAPIEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% endif %} +caCertPath: {{ kube_cert_dir }}/ca.crt +nodeRegistration: + name: {{ inventory_hostname }} +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% elif container_manager == 'rkt' %} + criSocket: /var/run/rkt.sock +{% else %} + criSocket: /var/run/dockershim.sock +{% endif %} diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml index c826d9c71af3428e3f2833482540d1c0ef8bec02..1ed2716360489a21199064aa57817bec98e68f16 100644 --- a/roles/kubernetes/master/tasks/kubeadm-setup.yml +++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml @@ -103,7 +103,14 @@ - name: sets kubeadm api version to v1alpha3 set_fact: kubeadmConfig_api_version: v1alpha3 - when: kubeadm_output.stdout is version('v1.12.0', '>=') + when: + - kubeadm_output.stdout is version('v1.12.0', '>=') + - kubeadm_output.stdout is version('v1.13.0', '<') + +- name: sets kubeadm api version to v1beta1 + set_fact: + kubeadmConfig_api_version: v1beta1 + when: kubeadm_output.stdout is version('v1.13.0', '>=') # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint. - name: set kubeadm_config_api_fqdn define @@ -144,15 +151,6 @@ failed_when: kubeadm_upgrade.rc != 0 and "field is immutable" not in kubeadm_upgrade.stderr notify: Master | restart kubelet -# FIXME(mattymo): remove when https://github.com/kubernetes/kubeadm/issues/433 is fixed -- name: kubeadm | Enable kube-proxy - command: "{{ bin_dir }}/kubeadm alpha phase addon kube-proxy --config={{ kube_config_dir }}/kubeadm-config.{{ kubeadmConfig_api_version }}.yaml" - register: kubeadm_kube_proxy_enable - retries: 10 - until: kubeadm_kube_proxy_enable is succeeded - when: inventory_hostname == groups['kube-master']|first - changed_when: false - - name: slurp kubeadm certs slurp: src: "{{ item }}" diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 index ee780cd5e0785c5093fd9023f96b31dfc436a07b..f2ad127c785ad3944de5205cdcc543c8121aacf3 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha1.yaml.j2 @@ -13,9 +13,9 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} - caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem - certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem - keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -69,6 +69,7 @@ apiServerExtraArgs: {% if kube_version is version('v1.9', '>=') %} endpoint-reconciler-type: lease {% endif %} + storage-backend: etcd3 {% if etcd_events_cluster_enabled %} etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 index 87e5a961ec1727478dbe65bd34595c9719dcb8ab..3385d2892c785ca927b856a2b21b690d624413c5 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 @@ -14,9 +14,9 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} - caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem - certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem - keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -54,6 +54,7 @@ apiServerExtraArgs: {% if kube_version is version('v1.9', '>=') %} endpoint-reconciler-type: lease {% endif %} + storage-backend: etcd3 {% if etcd_events_cluster_enabled %} etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 index 13053ae0b467291dc2f233d7a228d41c32d17bf5..d6f77ff7f99738f48e368b14c578c3951015b0d2 100644 --- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 +++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha3.yaml.j2 @@ -29,9 +29,9 @@ etcd: {% for endpoint in etcd_access_addresses.split(',') %} - {{ endpoint }} {% endfor %} - caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem - certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem - keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem networking: dnsDomain: {{ dns_domain }} serviceSubnet: {{ kube_service_addresses }} @@ -71,6 +71,7 @@ apiServerExtraArgs: {% if kube_version is version('v1.9', '>=') %} endpoint-reconciler-type: lease {% endif %} + storage-backend: etcd3 {% if etcd_events_cluster_enabled %} etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" {% endif %} diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..366cbee23613bd923ff16c5fcce1d0cc4975786e --- /dev/null +++ b/roles/kubernetes/master/templates/kubeadm-config.v1beta1.yaml.j2 @@ -0,0 +1,258 @@ +apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: {{ ip | default(ansible_default_ipv4.address) }} + bindPort: {{ kube_apiserver_port }} +nodeRegistration: +{% if kube_override_hostname|default('') %} + name: {{ kube_override_hostname }} +{% endif %} +{% if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] %} + taints: + - key: "kubeadmNode" + value: "master" + effect: "NoSchedule" +{% endif %} +{% if container_manager == 'crio' %} + criSocket: /var/run/crio/crio.sock +{% elif container_manager == 'rkt' %} + criSocket: /var/run/rkt.sock +{% else %} + criSocket: /var/run/dockershim.sock +{% endif %} +--- +apiVersion: kubeadm.k8s.io/v1beta1 +kind: ClusterConfiguration +clusterName: {{ cluster_name }} +etcd: + external: + endpoints: +{% for endpoint in etcd_access_addresses.split(',') %} + - {{ endpoint }} +{% endfor %} + caFile: {{ etcd_cert_dir }}/ca.pem + certFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}.pem + keyFile: {{ etcd_cert_dir }}/node-{{ inventory_hostname }}-key.pem +networking: + dnsDomain: {{ dns_domain }} + serviceSubnet: {{ kube_service_addresses }} + podSubnet: {{ kube_pods_subnet }} + podNetworkCidr: "{{ kube_network_node_prefix }}" +kubernetesVersion: {{ kube_version }} +{% if groups['kube-master'] | length > 1 and kubeadm_config_api_fqdn is defined %} +controlPlaneEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }} +{% else %} +controlPlaneEndpoint: {{ ip | default(ansible_default_ipv4.address) }}:{{ kube_apiserver_port }} +{% endif %} +certificatesDir: {{ kube_cert_dir }} +imageRepository: {{ kube_image_repo }} +UseHyperKubeImage: false +apiServer: + extraArgs: + authorization-mode: {{ authorization_modes | join(',') }} + bind-address: {{ kube_apiserver_bind_address }} +{% if kube_apiserver_insecure_port|string != "0" %} + insecure-bind-address: {{ kube_apiserver_insecure_bind_address }} +{% endif %} + insecure-port: "{{ kube_apiserver_insecure_port }}" +{% if kube_version is version('v1.10', '<') %} + admission-control: {{ kube_apiserver_admission_control | join(',') }} +{% else %} +{% if kube_apiserver_enable_admission_plugins|length > 0 %} + enable-admission-plugins: {{ kube_apiserver_enable_admission_plugins | join(',') }} +{% endif %} +{% if kube_apiserver_disable_admission_plugins|length > 0 %} + disable-admission-plugins: {{ kube_apiserver_disable_admission_plugins | join(',') }} +{% endif %} +{% endif %} + apiserver-count: "{{ kube_apiserver_count }}" +{% if kube_version is version('v1.9', '>=') %} + endpoint-reconciler-type: lease +{% endif %} + storage-backend: etcd3 +{% if etcd_events_cluster_enabled %} + etcd-servers-overrides: "/events#{{ etcd_events_access_addresses }}" +{% endif %} + service-node-port-range: {{ kube_apiserver_node_port_range }} + kubelet-preferred-address-types: "{{ kubelet_preferred_address_types }}" +{% if kube_basic_auth|default(true) %} + basic-auth-file: {{ kube_users_dir }}/known_users.csv +{% endif %} +{% if kube_token_auth|default(true) %} + token-auth-file: {{ kube_token_dir }}/known_tokens.csv +{% endif %} +{% if kube_oidc_auth|default(false) and kube_oidc_url is defined and kube_oidc_client_id is defined %} + oidc-issuer-url: {{ kube_oidc_url }} + oidc-client-id: {{ kube_oidc_client_id }} +{% if kube_oidc_ca_file is defined %} + oidc-ca-file: {{ kube_oidc_ca_file }} +{% endif %} +{% if kube_oidc_username_claim is defined %} + oidc-username-claim: {{ kube_oidc_username_claim }} +{% endif %} +{% if kube_oidc_groups_claim is defined %} + oidc-groups-claim: {{ kube_oidc_groups_claim }} +{% endif %} +{% endif %} +{% if kube_encrypt_secret_data %} + encryption-provider-config: {{ kube_config_dir }}/ssl/secrets_encryption.yaml +{% endif %} + storage-backend: {{ kube_apiserver_storage_backend }} +{% if kube_api_runtime_config is defined %} + runtime-config: {{ kube_api_runtime_config | join(',') }} +{% endif %} + allow-privileged: "true" +{% if kubernetes_audit %} + audit-log-path: "{{ audit_log_path }}" + audit-log-maxage: "{{ audit_log_maxage }}" + audit-log-maxbackup: "{{ audit_log_maxbackups }}" + audit-log-maxsize: "{{ audit_log_maxsize }}" + audit-policy-file: {{ audit_policy_file }} +{% endif %} +{% for key in kube_kubeadm_apiserver_extra_args %} + {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}" +{% endfor %} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kubernetes_audit or kube_basic_auth|default(true) or kube_token_auth|default(true) or ( cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] ) or apiserver_extra_volumes %} + extraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} + - name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if kube_basic_auth|default(true) %} + - name: basic-auth-config + hostPath: {{ kube_users_dir }} + mountPath: {{ kube_users_dir }} +{% endif %} +{% if kube_token_auth|default(true) %} + - name: token-auth-config + hostPath: {{ kube_token_dir }} + mountPath: {{ kube_token_dir }} +{% endif %} +{% if kubernetes_audit %} + - name: {{ audit_policy_name }} + hostPath: {{ audit_policy_hostpath }} + mountPath: {{ audit_policy_mountpath }} +{% if audit_log_path != "-" %} + - name: {{ audit_log_name }} + hostPath: {{ audit_log_hostpath }} + mountPath: {{ audit_log_mountpath }} + writable: true +{% endif %} +{% endif %} +{% for volume in apiserver_extra_volumes %} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + writable: {{ volume.writable | default(false)}} +{% endfor %} +{% endif %} + certSANs: +{% for san in apiserver_sans.split(' ') | unique %} + - {{ san }} +{% endfor %} + timeoutForControlPlane: 5m0s +controllerManager: + extraArgs: + node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }} + node-monitor-period: {{ kube_controller_node_monitor_period }} + pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }} +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% for key in kube_kubeadm_controller_extra_args %} + {{ key }}: "{{ kube_kubeadm_controller_extra_args[key] }}" +{% endfor %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws"] %} + cloud-provider: {{cloud_provider}} + cloud-config: {{ kube_config_dir }}/cloud_config +{% elif cloud_provider is defined and cloud_provider in ["external"] %} + cloud-config: {{ kube_config_dir }}/cloud_config +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] or controller_manager_extra_volumes %} + extraVolumes: +{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %} + - name: openstackcacert + hostPath: "{{ kube_config_dir }}/openstack-cacert.pem" + mountPath: "{{ kube_config_dir }}/openstack-cacert.pem" +{% endif %} +{% if cloud_provider is defined and cloud_provider in ["openstack", "azure", "vsphere", "aws", "external"] %} + - name: cloud-config + hostPath: {{ kube_config_dir }}/cloud_config + mountPath: {{ kube_config_dir }}/cloud_config +{% endif %} +{% for volume in controller_manager_extra_volumes %} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + writable: {{ volume.writable | default(false)}} +{% endfor %} +{% endif %} +scheduler: + extraArgs: +{% if kube_feature_gates %} + feature-gates: {{ kube_feature_gates|join(',') }} +{% endif %} +{% if kube_kubeadm_scheduler_extra_args|length > 0 %} +{% for key in kube_kubeadm_scheduler_extra_args %} + {{ key }}: "{{ kube_kubeadm_scheduler_extra_args[key] }}" +{% endfor %} +{% endif %} + extraVolumes: +{% if scheduler_extra_volumes %} + extraVolumes: +{% for volume in scheduler_extra_volumes %} + - name: {{ volume.name }} + hostPath: {{ volume.hostPath }} + mountPath: {{ volume.mountPath }} + writable: {{ volume.writable | default(false)}} +{% endfor %} +{% endif %} +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +bindAddress: 0.0.0.0 +clientConnection: + acceptContentTypes: "" + burst: 10 + contentType: application/vnd.kubernetes.protobuf + kubeconfig: /var/lib/kube-proxy/kubeconfig.conf + qps: 5 +clusterCIDR: "" +configSyncPeriod: 15m0s +conntrack: + max: null + maxPerCore: 32768 + min: 131072 + tcpCloseWaitTimeout: 1h0m0s + tcpEstablishedTimeout: 24h0m0s +enableProfiling: false +healthzBindAddress: 0.0.0.0:10256 +iptables: + masqueradeAll: false + masqueradeBit: 14 + minSyncPeriod: 0s + syncPeriod: 30s +ipvs: + excludeCIDRs: null + minSyncPeriod: 0s + scheduler: "" + syncPeriod: 30s +metricsBindAddress: 127.0.0.1:10249 +mode: {{ kube_proxy_mode }} +{% if kube_proxy_nodeport_addresses %} +nodePortAddresses: [{{ kube_proxy_nodeport_addresses_cidr }}] +{% endif %} +oomScoreAdj: -999 +portRange: "" +resourceContainer: "" +udpIdleTimeout: 250ms diff --git a/roles/kubespray-defaults/defaults/main.yaml b/roles/kubespray-defaults/defaults/main.yaml index 50d058626338d2f5b3100ee25c173163152ef5f1..745e2a9f81758215fe2a1218f6c0492ba0660298 100644 --- a/roles/kubespray-defaults/defaults/main.yaml +++ b/roles/kubespray-defaults/defaults/main.yaml @@ -12,7 +12,7 @@ is_atomic: false disable_swap: true ## Change this to use another Kubernetes version, e.g. a current beta release -kube_version: v1.12.3 +kube_version: v1.13.0 ## Kube Proxy mode One of ['iptables','ipvs'] kube_proxy_mode: ipvs diff --git a/roles/network_plugin/calico/templates/calico-node.yml.j2 b/roles/network_plugin/calico/templates/calico-node.yml.j2 index 2090372fbf46b5962de206c8ad028e7b0c770389..d8a433679607dec285d24682c5da111f7fbc2520 100644 --- a/roles/network_plugin/calico/templates/calico-node.yml.j2 +++ b/roles/network_plugin/calico/templates/calico-node.yml.j2 @@ -19,7 +19,6 @@ spec: k8s-app: calico-node annotations: # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' kubespray.etcd-cert/serial: "{{ etcd_client_cert_serial }}" {% if calico_felix_prometheusmetricsenabled %} prometheus.io/scrape: 'true' diff --git a/roles/network_plugin/canal/templates/canal-node.yaml.j2 b/roles/network_plugin/canal/templates/canal-node.yaml.j2 index 68a6b99102c87eb0fa13b86f69815704d859ae1b..a46608de869e3690c68be06b7bc89c5773f2c29c 100644 --- a/roles/network_plugin/canal/templates/canal-node.yaml.j2 +++ b/roles/network_plugin/canal/templates/canal-node.yaml.j2 @@ -12,9 +12,6 @@ spec: k8s-app: canal-node template: metadata: - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' labels: k8s-app: canal-node spec: diff --git a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 index b8a6306cf7d9a05b6b73c7e814e9297f3aa1e847..bda6000ae265e07b5ac7b85691760d30b1c38032 100755 --- a/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 +++ b/roles/network_plugin/cilium/templates/cilium-ds.yml.j2 @@ -21,12 +21,6 @@ spec: labels: k8s-app: cilium kubernetes.io/cluster-service: "true" - annotations: - # This annotation plus the CriticalAddonsOnly toleration makes - # cilium to be a critical pod in the cluster, which ensures cilium - # gets priority scheduling. - # https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ - scheduler.alpha.kubernetes.io/critical-pod: '' {% if cilium_enable_prometheus %} prometheus.io/scrape: "true" prometheus.io/port: "9090" diff --git a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 index 73a7688efa32bccdb7f6db3710965367820bcff8..c1604d0b5d8ec98d43940d295a97a2a7ed61f406 100644 --- a/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-api-proxy.yml.j2 @@ -15,9 +15,6 @@ spec: namespace: kube-system labels: k8s-app: contiv-api-proxy - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 index 73111072ca6d91a8cde343fad74d47395e3d7cbf..c8de9d29733be632ffc0655ec700e20323fcc9eb 100644 --- a/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-cleanup.yml.j2 @@ -14,9 +14,6 @@ spec: metadata: labels: k8s-app: contiv-cleanup - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 index 0b67945e534eefe188565575215d9938185076e1..a16ee5755b5ec8428e1290e2d6809bd0e8c046cc 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd-proxy.yml.j2 @@ -14,8 +14,6 @@ spec: metadata: labels: k8s-app: contiv-etcd-proxy - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 index b8fba9cc6bdd7d5831bbed3b458d43ad089a84c7..e320f5b24d051c78222b1e67fe15d137033def0b 100644 --- a/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-etcd.yml.j2 @@ -14,8 +14,6 @@ spec: metadata: labels: k8s-app: contiv-etcd - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 index e615102534c6f0e9c96509963c28b93ca1b1073e..a39938f77df2639f7ca53fcb4fc25f2b19f7770e 100644 --- a/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netmaster.yml.j2 @@ -15,9 +15,6 @@ spec: namespace: kube-system labels: k8s-app: contiv-netmaster - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 index 045b9e7eb281ff8e9b27688dbe53b16441569f81..8b2e65ebd5619e2919cbd6f16ce84ada312eb241 100644 --- a/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-netplugin.yml.j2 @@ -19,9 +19,6 @@ spec: metadata: labels: k8s-app: contiv-netplugin - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 index 40c37b6ad75a17c8e3e010aedbebd23c2d9df6f4..2ec15fc825cae1c0f882af988e9bc9659c9fedab 100644 --- a/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 +++ b/roles/network_plugin/contiv/templates/contiv-ovs.yml.j2 @@ -16,9 +16,6 @@ spec: metadata: labels: k8s-app: contiv-ovs - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 index 578409d029f1b7a4b7bdbc4faeb7fc342d6c122b..bcaae4a6d76b1c5fa4e125a4486e8b460ca9b537 100644 --- a/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 +++ b/roles/network_plugin/flannel/templates/cni-flannel.yml.j2 @@ -51,9 +51,6 @@ spec: labels: tier: node k8s-app: flannel - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical diff --git a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 b/roles/network_plugin/kube-router/templates/kube-router.yml.j2 index ac1029a0ea91faa85ffa1732867082812507562a..37f03ea26a7b07a048bcf868bc48b806ce6a5b79 100644 --- a/roles/network_plugin/kube-router/templates/kube-router.yml.j2 +++ b/roles/network_plugin/kube-router/templates/kube-router.yml.j2 @@ -60,8 +60,6 @@ spec: labels: k8s-app: kube-router tier: node - annotations: - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-cluster-critical diff --git a/roles/network_plugin/weave/templates/weave-net.yml.j2 b/roles/network_plugin/weave/templates/weave-net.yml.j2 index 204e3f993ea2d123685814c8b7f942d4086cf611..3d66c043c6ed72a06338e6aa64f6fcdfb069eb3d 100644 --- a/roles/network_plugin/weave/templates/weave-net.yml.j2 +++ b/roles/network_plugin/weave/templates/weave-net.yml.j2 @@ -114,9 +114,6 @@ items: metadata: labels: name: weave-net - annotations: - # Mark pod as critical for rescheduling (Will have no effect starting with kubernetes 1.12) - scheduler.alpha.kubernetes.io/critical-pod: '' spec: {% if kube_version is version('v1.11.1', '>=') %} priorityClassName: system-node-critical