diff --git a/README.md b/README.md index 9a234fd0cb9d426de7669d900c4c73e326cc3fd4..9685f375a288f6d58c9f6fa6dcc09709ed2bf5c9 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ Supported Components - Application - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v1.1.0-k8s1.10 - [cert-manager](https://github.com/jetstack/cert-manager) v0.3.2 - - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.15.0 + - [ingress-nginx](https://github.com/kubernetes/ingress-nginx) v0.16.2 Note: kubernetes doesn't support newer docker versions. Among other things kubelet currently breaks on docker's non-standard version numbering (it no longer uses semantic versioning). To ensure auto-updates don't break your cluster look into e.g. yum versionlock plugin or apt pin). diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml index 20805d0c1dd530ffe84b921adf09a29e1f91dd62..cc77d50089388cdcc91d1eb4cde1042ce485ac4a 100644 --- a/inventory/sample/group_vars/k8s-cluster.yml +++ b/inventory/sample/group_vars/k8s-cluster.yml @@ -208,6 +208,8 @@ cephfs_provisioner_enabled: false # Nginx ingress controller deployment ingress_nginx_enabled: false # ingress_nginx_host_network: false +# ingress_nginx_nodeselector: +# node-role.kubernetes.io/master: "true" # ingress_nginx_namespace: "ingress-nginx" # ingress_nginx_insecure_port: 80 # ingress_nginx_secure_port: 443 diff --git a/inventory/sample/hosts.ini b/inventory/sample/hosts.ini index bddfa2f80a2f28f2319745b8a17f0bf7a3ed4d33..ad38aedf2ba474408f5e7b3038e42df168747170 100644 --- a/inventory/sample/hosts.ini +++ b/inventory/sample/hosts.ini @@ -26,11 +26,6 @@ # node5 # node6 -# [kube-ingress] -# node2 -# node3 - # [k8s-cluster:children] # kube-master # kube-node -# kube-ingress diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml index 597eea501a1365c80f8aef63b9a32748e31d1b93..2e7937f98b6010d2b9d8c2d332c1e05da1108ddc 100644 --- a/roles/download/defaults/main.yml +++ b/roles/download/defaults/main.yml @@ -157,7 +157,7 @@ local_volume_provisioner_image_tag: "v2.0.0" cephfs_provisioner_image_repo: "quay.io/external_storage/cephfs-provisioner" cephfs_provisioner_image_tag: "v1.1.0-k8s1.10" ingress_nginx_controller_image_repo: "quay.io/kubernetes-ingress-controller/nginx-ingress-controller" -ingress_nginx_controller_image_tag: "0.15.0" +ingress_nginx_controller_image_tag: "0.16.2" ingress_nginx_default_backend_image_repo: "gcr.io/google_containers/defaultbackend" ingress_nginx_default_backend_image_tag: "1.4" cert_manager_version: "v0.3.2" @@ -564,7 +564,7 @@ downloads: tag: "{{ ingress_nginx_controller_image_tag }}" sha256: "{{ ingress_nginx_controller_digest_checksum|default(None) }}" groups: - - kube-ingress + - kube-node ingress_nginx_default_backend: enabled: "{{ ingress_nginx_enabled }}" container: true @@ -572,7 +572,7 @@ downloads: tag: "{{ ingress_nginx_default_backend_image_tag }}" sha256: "{{ ingress_nginx_default_backend_digest_checksum|default(None) }}" groups: - - kube-ingress + - kube-node cert_manager_controller: enabled: "{{ cert_manager_enabled }}" container: true diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml index ff12178092aed6b814b0c957b75cbff1ad3784f0..8acee53eb64fc6927cadf732ac3fb26d3420154f 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/defaults/main.yml @@ -1,6 +1,8 @@ --- ingress_nginx_namespace: "ingress-nginx" ingress_nginx_host_network: false +ingress_nginx_nodeselector: + node-role.kubernetes.io/master: "true" ingress_nginx_insecure_port: 80 ingress_nginx_secure_port: 443 ingress_nginx_configmap: {} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml index 0a37e94cdd7b40b6edc7dd5226fe231901a3ef0b..eff3c7ed8ad2acc3f196ec112aef6259e18a6ec8 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/tasks/main.yml @@ -1,5 +1,23 @@ --- +- name: NGINX Ingress Controller | Remove legacy addon dir and manifests + file: + path: "{{ kube_config_dir }}/addons/ingress_nginx" + state: absent + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + +- name: NGINX Ingress Controller | Remove legacy namespace + shell: | + {{ bin_dir }}/kubectl delete namespace {{ ingress_nginx_namespace }} + ignore_errors: yes + when: + - inventory_hostname == groups['kube-master'][0] + tags: + - upgrade + - name: NGINX Ingress Controller | Create addon dir file: path: "{{ kube_config_dir }}/addons/ingress_nginx" @@ -7,24 +25,26 @@ owner: root group: root mode: 0755 + when: + - inventory_hostname == groups['kube-master'][0] - name: NGINX Ingress Controller | Create manifests template: src: "{{ item.file }}.j2" dest: "{{ kube_config_dir }}/addons/ingress_nginx/{{ item.file }}" with_items: - - { name: ingress-nginx-ns, file: ingress-nginx-ns.yml, type: ns } - - { name: ingress-nginx-sa, file: ingress-nginx-sa.yml, type: sa } - - { name: ingress-nginx-role, file: ingress-nginx-role.yml, type: role } - - { name: ingress-nginx-rolebinding, file: ingress-nginx-rolebinding.yml, type: rolebinding } - - { name: ingress-nginx-clusterrole, file: ingress-nginx-clusterrole.yml, type: clusterrole } - - { name: ingress-nginx-clusterrolebinding, file: ingress-nginx-clusterrolebinding.yml, type: clusterrolebinding } - - { name: ingress-nginx-cm, file: ingress-nginx-cm.yml, type: cm } - - { name: ingress-nginx-tcp-servicecs-cm, file: ingress-nginx-tcp-servicecs-cm.yml, type: cm } - - { name: ingress-nginx-udp-servicecs-cm, file: ingress-nginx-udp-servicecs-cm.yml, type: cm } - - { name: ingress-nginx-default-backend-svc, file: ingress-nginx-default-backend-svc.yml, type: svc } - - { name: ingress-nginx-default-backend-rs, file: ingress-nginx-default-backend-rs.yml, type: rs } - - { name: ingress-nginx-controller-ds, file: ingress-nginx-controller-ds.yml, type: ds } + - { name: 00-namespace, file: 00-namespace.yml, type: ns } + - { name: deploy-default-backend, file: deploy-default-backend.yml, type: deploy } + - { name: svc-default-backend, file: svc-default-backend.yml, type: svc } + - { name: cm-ingress-nginx, file: cm-ingress-nginx.yml, type: cm } + - { name: cm-tcp-services, file: cm-tcp-services.yml, type: cm } + - { name: cm-udp-services, file: cm-udp-services.yml, type: cm } + - { name: sa-ingress-nginx, file: sa-ingress-nginx.yml, type: sa } + - { name: clusterrole-ingress-nginx, file: clusterrole-ingress-nginx.yml, type: clusterrole } + - { name: clusterrolebinding-ingress-nginx, file: clusterrolebinding-ingress-nginx.yml, type: clusterrolebinding } + - { name: role-ingress-nginx, file: role-ingress-nginx.yml, type: role } + - { name: rolebinding-ingress-nginx, file: rolebinding-ingress-nginx.yml, type: rolebinding } + - { name: ds-ingress-nginx-controller, file: ds-ingress-nginx-controller.yml, type: ds } register: ingress_nginx_manifests when: - inventory_hostname == groups['kube-master'][0] diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-ns.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-ns.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/00-namespace.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrole.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrole.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrole-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrolebinding.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-clusterrolebinding.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/clusterrolebinding-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 similarity index 82% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 index 7e47e81b13cae7655ed70535f96293d2d196721f..00c44a97beed98a94d78b46acb08e7140dffe84d 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-ingress-nginx.yml.j2 @@ -6,5 +6,7 @@ metadata: namespace: {{ ingress_nginx_namespace }} labels: k8s-app: ingress-nginx +{% if ingress_nginx_configmap %} data: {{ ingress_nginx_configmap | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 similarity index 71% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 index 0a87e91b7433c564e8730a9805aa920fe2666bf5..d97c42d97dee243466870716d8927c2f2e454483 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-tcp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-tcp-services.yml.j2 @@ -2,9 +2,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ingress-nginx-tcp-services + name: tcp-services namespace: {{ ingress_nginx_namespace }} labels: k8s-app: ingress-nginx +{% if ingress_nginx_configmap_tcp_services %} data: {{ ingress_nginx_configmap_tcp_services | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 similarity index 71% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 index d943e57185cac2cdd1392df31624d0e2f388c619..b343869b7941736ea6f99185877766e3def78aca 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-udp-servicecs-cm.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/cm-udp-services.yml.j2 @@ -2,9 +2,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ingress-nginx-udp-services + name: udp-services namespace: {{ ingress_nginx_namespace }} labels: k8s-app: ingress-nginx +{% if ingress_nginx_configmap_udp_services %} data: {{ ingress_nginx_configmap_udp_services | to_nice_yaml | indent(2) }} +{%- endif %} diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-rs.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 similarity index 71% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-rs.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 index c0bed920b25fd6511a6d4e2f45f4c694c1eadad7..eca5a50845c1c682bf4cb454c9fd37fa445b91a4 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-rs.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/deploy-default-backend.yml.j2 @@ -1,27 +1,27 @@ --- apiVersion: apps/v1 -kind: ReplicaSet +kind: Deployment metadata: - name: ingress-nginx-default-backend-v{{ ingress_nginx_default_backend_image_tag }} + name: default-backend-v{{ ingress_nginx_default_backend_image_tag }} namespace: {{ ingress_nginx_namespace }} labels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend version: v{{ ingress_nginx_default_backend_image_tag }} spec: replicas: 1 selector: matchLabels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend version: v{{ ingress_nginx_default_backend_image_tag }} template: metadata: labels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend version: v{{ ingress_nginx_default_backend_image_tag }} spec: terminationGracePeriodSeconds: 60 containers: - - name: ingress-nginx-default-backend + - name: default-backend # Any image is permissible as long as: # 1. It serves a 404 page at / # 2. It serves 200 on a /healthz endpoint @@ -35,3 +35,10 @@ spec: timeoutSeconds: 5 ports: - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 similarity index 79% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 index 40e1d4715037b052627df59b53f345499ea51bba..5d141d4ffdee67fb9b2b57129b3dd6ef950899f7 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-controller-ds.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ds-ingress-nginx-controller.yml.j2 @@ -7,9 +7,6 @@ metadata: labels: k8s-app: ingress-nginx version: v{{ ingress_nginx_controller_image_tag }} - annotations: - prometheus.io/port: '10254' - prometheus.io/scrape: 'true' spec: selector: matchLabels: @@ -24,23 +21,36 @@ spec: prometheus.io/port: '10254' prometheus.io/scrape: 'true' spec: +{% if rbac_enabled %} + serviceAccountName: ingress-nginx +{% endif %} {% if ingress_nginx_host_network %} hostNetwork: true {% endif %} +{% if ingress_nginx_nodeselector %} nodeSelector: - node-role.kubernetes.io/ingress: "true" - terminationGracePeriodSeconds: 60 + {{ ingress_nginx_nodeselector | to_nice_yaml }} +{%- endif %} containers: - name: ingress-nginx-controller image: {{ ingress_nginx_controller_image_repo }}:{{ ingress_nginx_controller_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} args: - /nginx-ingress-controller - - --default-backend-service=$(POD_NAMESPACE)/ingress-nginx-default-backend + - --default-backend-service=$(POD_NAMESPACE)/default-backend - --configmap=$(POD_NAMESPACE)/ingress-nginx - - --tcp-services-configmap=$(POD_NAMESPACE)/ingress-nginx-tcp-services - - --udp-services-configmap=$(POD_NAMESPACE)/ingress-nginx-udp-services + - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services + - --udp-services-configmap=$(POD_NAMESPACE)/udp-services + - --publish-service=$(POD_NAMESPACE)/ingress-nginx - --annotations-prefix=nginx.ingress.kubernetes.io + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + # www-data -> 33 + runAsUser: 33 env: - name: POD_NAME valueFrom: @@ -78,7 +88,3 @@ spec: timeoutSeconds: 1 securityContext: runAsNonRoot: false -{% if rbac_enabled %} - serviceAccountName: ingress-nginx -{% endif %} - diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-role.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-role.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/role-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-rolebinding.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-rolebinding.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/rolebinding-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-sa.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 similarity index 100% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-sa.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/sa-ingress-nginx.yml.j2 diff --git a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-svc.yml.j2 b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2 similarity index 56% rename from roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-svc.yml.j2 rename to roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2 index ab23f37995976bcc3c60f33cbe697584ed626f8b..326cc884397c2c75d45e10edb4042abac2761542 100644 --- a/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/ingress-nginx-default-backend-svc.yml.j2 +++ b/roles/kubernetes-apps/ingress_controller/ingress_nginx/templates/svc-default-backend.yml.j2 @@ -2,13 +2,13 @@ apiVersion: v1 kind: Service metadata: - name: ingress-nginx-default-backend + name: default-backend namespace: {{ ingress_nginx_namespace }} labels: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend spec: ports: - port: 80 targetPort: 8080 selector: - k8s-app: ingress-nginx-default-backend + k8s-app: default-backend diff --git a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 index 4ca17ef5322ec00a63e4196e436a36d73b1fadea..e313161a080e9cf16058bb9c5bab40e16d544fdf 100644 --- a/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2 @@ -75,9 +75,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} {% set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} -{% if inventory_hostname in groups['kube-ingress']|default([]) %} -{% set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %} -{% endif %} {% set inventory_node_labels = [] %} {% if node_labels is defined %} {% for labelname, labelvalue in node_labels.iteritems() %} diff --git a/roles/kubernetes/node/templates/kubelet.standard.env.j2 b/roles/kubernetes/node/templates/kubelet.standard.env.j2 index 83d657f7e03c8293d2cb3905b1c7dbfe87a99416..96fad4d0e72635bd2b10708fc84b54806708a6b4 100644 --- a/roles/kubernetes/node/templates/kubelet.standard.env.j2 +++ b/roles/kubernetes/node/templates/kubelet.standard.env.j2 @@ -91,9 +91,6 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}" {% else %} {% set dummy = role_node_labels.append('node-role.kubernetes.io/node=true') %} {% endif %} -{% if inventory_hostname in groups['kube-ingress']|default([]) %} -{% set dummy = role_node_labels.append('node-role.kubernetes.io/ingress=true') %} -{% endif %} {% set inventory_node_labels = [] %} {% if node_labels is defined %} {% for labelname, labelvalue in node_labels.iteritems() %}