diff --git a/docs/security.md b/docs/security.md deleted file mode 100644 index fd45a579eee99db4a6f14c2010ffd838f0df658d..0000000000000000000000000000000000000000 --- a/docs/security.md +++ /dev/null @@ -1,31 +0,0 @@ -Users and groups -================ - -There are following users and groups defined by the addusers role: - -* Kube user, group from the ``kubelet_user`` and ``kubelet_group`` vars. -* Etcd user, group from the ``etcd_user`` and ``etcd_group`` vars. -* Network plugin user, group from the ``netplug_user`` and ``netplug_group`` vars. - -There are additional certificate access groups for kube and etcd users defined. -For example, kubelet and network plugins require read access to the -etcd certs and keys. This is defined via the corresponding ``etcd_cert_group`` -var. Members of that group (defaults to `kube` and `netplug` users) will read -etcd secret keys and certs. Same applies to the ``kube_cert_group`` -(defaults to `kube` user) members. You may want to share kube certs via that -group with bastion proxies or the like. - -Linux capabilites -================= - -Kargo allows to control dropped Linux capabilities for unprivileged docker -containers it configures for deployments. For examle, etcd or some networking -related systemd units or k8s workloads, like kubedns, dnsmasq or netchecker apps. - -Dropped capabilites are represented by the ``apps_drop_cap``, ``dnsmasq_drop_cap``, -``etcd_drop_cap``, ``calico_drop_cap`` vars. - -Be carefull changing defaults - different kube components and k8s apps might -expect specific capabilities to be present and can only run as root! Also note -that kublet, kube-proxy and network plugins require privileged mode and ignore -dropped capabilities. diff --git a/inventory/group_vars/all.yml b/inventory/group_vars/all.yml index 3c0a5b9d7b1c9095d239d0c9a048831eacecf104..e5047ca4791645a5a183baabc2a6997712a27776 100644 --- a/inventory/group_vars/all.yml +++ b/inventory/group_vars/all.yml @@ -36,19 +36,10 @@ retry_stagger: 5 # Directory where python binary is installed # ansible_python_interpreter: "/opt/bin/python" -# This is the users/groups to own files and groups that the cert creation -# scripts chgrp the cert files to -kubelet_user: kube -kubelet_group: kube +# This is the group that the cert creation scripts chgrp the +# cert files to. Not really changable... kube_cert_group: kube-cert -netplug_user: netplug -netplug_group: netplug - -etcd_user: etcd -etcd_group: etcd -etcd_cert_group: etcd-cert - # Cluster Loglevel configuration kube_log_level: 2 diff --git a/roles/adduser/defaults/main.yml b/roles/adduser/defaults/main.yml index 3766b2c0dc44ed1c529d88f411c9a7fbd75778a3..b3a69229c15aca990cd255d6c0f0ddf68a4f9daa 100644 --- a/roles/adduser/defaults/main.yml +++ b/roles/adduser/defaults/main.yml @@ -1,40 +1,24 @@ --- addusers: etcd: - name: "{{ etcd_user }}" + name: etcd comment: "Etcd user" - createhome: >- - {% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %}no{% else %}yes{% endif %} + createhome: yes home: "/var/lib/etcd" system: yes - shell: /usr/sbin/nologin - group: "{{ etcd_group }}" - groups: "{{ etcd_cert_group }}" - type: >- - {% if ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] %}cloud-init{% endif %} + shell: /bin/nologin kube: - name: "{{ kubelet_user }}" + name: kube comment: "Kubernetes user" - shell: /usr/sbin/nologin + shell: /sbin/nologin system: yes - group: "{{ kubelet_group }}" - groups: "{{ etcd_cert_group }},{{ kube_cert_group }}" + group: "{{ kube_cert_group }}" createhome: no - netplug: - name: "{{ netplug_user }}" - comment: "Network plugin user" - createhome: no - system: yes - shell: /usr/sbin/nologin - group: "{{ netplug_group }}" - groups: "{{ etcd_cert_group }}" adduser: name: "{{ user.name }}" group: "{{ user.name|default(None) }}" - groups: "{{ user.groups|default(None) }}" comment: "{{ user.comment|default(None) }}" shell: "{{ user.shell|default(None) }}" system: "{{ user.system|default(None) }}" createhome: "{{ user.createhome|default(None) }}" - type: "{{ user.type|default(None) }}" diff --git a/roles/adduser/handlers/main.yml b/roles/adduser/handlers/main.yml deleted file mode 100644 index 60d821f181f51204d5606b3fc2081b386e6c8898..0000000000000000000000000000000000000000 --- a/roles/adduser/handlers/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: User | update users for cloud-init - command: /usr/bin/coreos-cloudinit --from-file /etc/{{ user.name }}_user_cloud_init_conf - when: ansible_os_family in ["CoreOS", "Container Linux by CoreOS"] diff --git a/roles/adduser/tasks/main.yml b/roles/adduser/tasks/main.yml index 8176a5c7134248ad72beae09676e7828887b7a13..394ff92945c99882977c63b4e668131b13189fb3 100644 --- a/roles/adduser/tasks/main.yml +++ b/roles/adduser/tasks/main.yml @@ -1,35 +1,13 @@ --- -- name: User | Create Certificate Access Groups - group: name={{ item }} system=yes - with_items: "{{ user.groups.split(',') }}" - - name: User | Create User Group group: name={{user.group|default(user.name)}} system={{user.system|default(omit)}} -- name: User | Create cloud-init user - template: - dest: /etc/{{ user.name }}_user_cloud_init_conf - src: users.j2 - owner: root - mode: 0640 - notify: User | update users for cloud-init - when: "{{ user.type|default('standard') == 'cloud-init' }}" - -- meta: flush_handlers - -- name: User | Hack groups for existing cloud-init users CoreOS - command: /usr/sbin/usermod -aG {{ item }} {{ user.name }} - with_items: "{{ user.groups.split(',') }}" - when: "{{ ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] and user.type|default('standard') == 'cloud-init' }}" - - name: User | Create User user: comment: "{{user.comment|default(omit)}}" - createhome: "{{user.createhome|default(omit)}}" + createhome: "{{user.create_home|default(omit)}}" group: "{{user.group|default(user.name)}}" - groups: "{{user.groups|default(omit)}}" home: "{{user.home|default(omit)}}" shell: "{{user.shell|default(omit)}}" name: "{{user.name}}" system: "{{user.system|default(omit)}}" - when: "{{ user.type|default('standard') != 'cloud-init' }}" diff --git a/roles/adduser/templates/users.j2 b/roles/adduser/templates/users.j2 deleted file mode 100644 index 345049dc725df88abd5d72c0b868cbfe00124774..0000000000000000000000000000000000000000 --- a/roles/adduser/templates/users.j2 +++ /dev/null @@ -1,15 +0,0 @@ -#cloud-config -users: -- name: {{ user.name }} - gecos: {{ user.comment }} - system: {{ user.system|bool }} - no-log-init: {{ user.system|bool }} - primary-group: {{ user.group }} - no-create-home: {{ not user.createhome|bool }} - homedir: {{ user.home }} - shell: {{ user.shell }} - groups: | - {% for g in user.groups.split(',') %} - - {{ g }} - {% endfor %} - # diff --git a/roles/adduser/vars/coreos.yml b/roles/adduser/vars/coreos.yml new file mode 100644 index 0000000000000000000000000000000000000000..9fa93e45bf3919d6c1d95146ffdb93f181f98eef --- /dev/null +++ b/roles/adduser/vars/coreos.yml @@ -0,0 +1,8 @@ +--- +addusers: + - name: kube + comment: "Kubernetes user" + shell: /sbin/nologin + system: yes + group: "{{ kube_cert_group }}" + createhome: no diff --git a/roles/adduser/vars/debian.yml b/roles/adduser/vars/debian.yml new file mode 100644 index 0000000000000000000000000000000000000000..16b39f656ec222e3134492e37e064bc965ccb6f6 --- /dev/null +++ b/roles/adduser/vars/debian.yml @@ -0,0 +1,15 @@ +--- +addusers: + - name: etcd + comment: "Etcd user" + createhome: yes + home: "/var/lib/etcd" + system: yes + shell: /bin/nologin + + - name: kube + comment: "Kubernetes user" + shell: /sbin/nologin + system: yes + group: "{{ kube_cert_group }}" + createhome: no diff --git a/roles/adduser/vars/redhat.yml b/roles/adduser/vars/redhat.yml new file mode 100644 index 0000000000000000000000000000000000000000..16b39f656ec222e3134492e37e064bc965ccb6f6 --- /dev/null +++ b/roles/adduser/vars/redhat.yml @@ -0,0 +1,15 @@ +--- +addusers: + - name: etcd + comment: "Etcd user" + createhome: yes + home: "/var/lib/etcd" + system: yes + shell: /bin/nologin + + - name: kube + comment: "Kubernetes user" + shell: /sbin/nologin + system: yes + group: "{{ kube_cert_group }}" + createhome: no diff --git a/roles/dnsmasq/defaults/main.yml b/roles/dnsmasq/defaults/main.yml index 874e636c83a885cc0fc2b28ff7e821d0233df930..d8ac8b34be8275c118aa93cb91abc537208636f0 100644 --- a/roles/dnsmasq/defaults/main.yml +++ b/roles/dnsmasq/defaults/main.yml @@ -26,16 +26,3 @@ dns_cpu_limit: 100m dns_memory_limit: 170Mi dns_cpu_requests: 70m dns_memory_requests: 70Mi - -# Linux capabilities to be dropped for dnsmasq k8s app ran container engines -dnsmasq_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap diff --git a/roles/dnsmasq/templates/dnsmasq-ds.yml b/roles/dnsmasq/templates/dnsmasq-ds.yml index 97d809832004bd6e9f7a677ec16763bdac6d1295..adcbbeacbd7130f4b7af46d2f311673d6f43860e 100644 --- a/roles/dnsmasq/templates/dnsmasq-ds.yml +++ b/roles/dnsmasq/templates/dnsmasq-ds.yml @@ -26,10 +26,6 @@ spec: capabilities: add: - NET_ADMIN - drop: -{% for c in dnsmasq_drop_cap %} - - {{ c.upper() }} -{% endfor %} imagePullPolicy: IfNotPresent resources: limits: diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml index 14b41edeb572b7fce417850bbc00e56e8d8f77cd..e733fe56d9107b508523d63694d4b303afa5990f 100644 --- a/roles/etcd/defaults/main.yml +++ b/roles/etcd/defaults/main.yml @@ -3,26 +3,10 @@ etcd_bin_dir: "{{ local_release_dir }}/etcd/etcd-{{ etcd_version }}-linux-amd64/ etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" +etcd_cert_group: root etcd_script_dir: "{{ bin_dir }}/etcd-scripts" -# Linux capabilities to be dropped for container engines -etcd_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - net_bind_service - - net_raw - - sys_chroot - - mknod - - audit_write - - setfcap - # Limits etcd_memory_limit: 512M etcd_cpu_limit: 300m diff --git a/roles/etcd/files/make-ssl-etcd.sh b/roles/etcd/files/make-ssl-etcd.sh index da76e3f55fd414b64ddcb712abce026cff8ec1d7..458b39d9bc95bb3fd389af917c8698d7a15535b4 100755 --- a/roles/etcd/files/make-ssl-etcd.sh +++ b/roles/etcd/files/make-ssl-etcd.sh @@ -94,8 +94,5 @@ if [ -n "$HOSTS" ]; then done fi -# Grant the group read access -chmod g+r *.pem - # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/etcd/meta/main.yml b/roles/etcd/meta/main.yml index fbf654981a3b9f750d02571c62a2ff46c9df2da2..addd81053d3da495880454c5a177bee33209923d 100644 --- a/roles/etcd/meta/main.yml +++ b/roles/etcd/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - role: adduser user: "{{ addusers.etcd }}" - tags: bootstrap-os + when: not ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] - role: download file: "{{ downloads.etcd }}" tags: download diff --git a/roles/etcd/tasks/gen_certs.yml b/roles/etcd/tasks/gen_certs.yml index 49a01ac347b29fcadb12469b51a66ebc4d52d9af..a4fd3a9d79eccb0cc502f34bafb68420761fe3c0 100644 --- a/roles/etcd/tasks/gen_certs.yml +++ b/roles/etcd/tasks/gen_certs.yml @@ -4,15 +4,14 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - mode=0750 - owner={{ etcd_user }} + owner=root recurse=yes - name: "Gen_certs | create etcd script dir (on {{groups['etcd'][0]}})" file: path: "{{ etcd_script_dir }}" state: directory - owner: "{{ etcd_user }}" + owner: root run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -21,8 +20,7 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - mode=0750 - owner={{ etcd_user }} + owner=root recurse=yes run_once: yes delegate_to: "{{groups['etcd'][0]}}" @@ -126,12 +124,12 @@ path={{ etcd_cert_dir }} group={{ etcd_cert_group }} state=directory - owner={{ etcd_user }} + owner=kube recurse=yes tags: facts -- name: Gen_certs | set shared group permissions on keys - shell: chmod 0640 {{ etcd_cert_dir}}/*.pem +- name: Gen_certs | set permissions on keys + shell: chmod 0600 {{ etcd_cert_dir}}/*key.pem when: inventory_hostname in groups['etcd'] changed_when: false diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml index c9a662d6c984dde6b301b08e06eb587a5d55f54c..394e5de6435050793cf1adfd7148846dd97e71cf 100644 --- a/roles/etcd/tasks/main.yml +++ b/roles/etcd/tasks/main.yml @@ -1,8 +1,6 @@ --- - include: pre_upgrade.yml tags: etcd-pre-upgrade -- include: set_facts.yml - tags: [bootstrap-os, facts] - include: check_certs.yml tags: [etcd-secrets, facts] - include: gen_certs.yml diff --git a/roles/etcd/tasks/pre_upgrade.yml b/roles/etcd/tasks/pre_upgrade.yml index 30f307a037576ce4656c3b9dc9f27ac77b468c4b..eb17e987114624d1baea59b73948a6fbde66b85f 100644 --- a/roles/etcd/tasks/pre_upgrade.yml +++ b/roles/etcd/tasks/pre_upgrade.yml @@ -1,4 +1,3 @@ ---- - name: "Pre-upgrade | check for etcd-proxy unit file" stat: path: /etc/systemd/system/etcd-proxy.service @@ -50,7 +49,3 @@ awk -F"[: =]" '{print "{{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses | regex_replace('https','http') }} member update "$1" https:"$7":"$8}' | bash run_once: true when: 'etcd_member_list.rc == 0 and "http://" in etcd_member_list.stdout' - -- name: "Pre-upgrade | share access to etcd certs for its users" - shell: chmod g+r {{ etcd_cert_dir }}/*.pem - failed_when: false diff --git a/roles/etcd/tasks/set_facts.yml b/roles/etcd/tasks/set_facts.yml deleted file mode 100644 index 1d5f20462685b64ef5c7089eec2f96a9c266f779..0000000000000000000000000000000000000000 --- a/roles/etcd/tasks/set_facts.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Etcd | get etcd user ID - shell: /usr/bin/id -u {{ etcd_user }} || echo 0 - register: etcd_uid - -- name: Etcd | get etcd group ID - shell: /usr/bin/getent group {{ etcd_group }} | cut -d':' -f3 || echo 0 - register: etcd_gid - -- name: Etcd | get etcd cert group ID - shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0 - register: etcd_cert_gid - -- set_fact: - etcd_user_id: "{{ etcd_uid.stdout }}" - etcd_group_id: "{{ etcd_gid.stdout }}" - etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" diff --git a/roles/etcd/templates/etcd-docker.service.j2 b/roles/etcd/templates/etcd-docker.service.j2 index 05640b14688e84431fa0fd2244355074101f8703..223d2d8427d89c9cb2617e48556eb2f065579729 100644 --- a/roles/etcd/templates/etcd-docker.service.j2 +++ b/roles/etcd/templates/etcd-docker.service.j2 @@ -14,12 +14,8 @@ ExecStart={{ docker_bin_dir }}/docker run --restart=on-failure:5 \ -v /etc/ssl/certs:/etc/ssl/certs:ro \ -v {{ etcd_cert_dir }}:{{ etcd_cert_dir }}:ro \ -v /var/lib/etcd:/var/lib/etcd:rw \ -{% for c in etcd_drop_cap %} ---cap-drop={{ c }} \ -{% endfor %} --memory={{ etcd_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ etcd_cpu_limit|regex_replace('m', '') }} \ --name={{ etcd_member_name | default("etcd") }} \ --u {{ etcd_user_id }}:{{ etcd_group_id }} --group-add {{ etcd_cert_group_id }} \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ {% if etcd_after_v3 %} {{ etcd_container_bin_dir }}etcd diff --git a/roles/etcd/templates/etcd-rkt.service.j2 b/roles/etcd/templates/etcd-rkt.service.j2 index 72e98d21e9b17c78cadedd07c1394502b7953a2e..eb26bc473ed8062597ce4cedda2022146e33ca7b 100644 --- a/roles/etcd/templates/etcd-rkt.service.j2 +++ b/roles/etcd/templates/etcd-rkt.service.j2 @@ -8,9 +8,6 @@ Restart=on-failure RestartSec=10s TimeoutStartSec=0 LimitNOFILE=40000 -User=root -Group={{ etcd_group_id }} -SupplementaryGroups={{ etcd_cert_group_id }} ExecStart=/usr/bin/rkt run \ --uuid-file-save=/var/run/etcd.uuid \ @@ -23,11 +20,6 @@ ExecStart=/usr/bin/rkt run \ --set-env-file=/etc/etcd.env \ --stage1-from-dir=stage1-fly.aci \ {{ etcd_image_repo }}:{{ etcd_image_tag }} \ -{% for c in etcd_drop_cap %} ---caps-remove=CAP_{{ c.upper() }} \ -{% endfor %} ---memory={{ etcd_memory_limit }} --cpu={{ etcd_cpu_limit }} \ ---user={{ etcd_user_id }} --group={{ etcd_group_id }} \ --name={{ etcd_member_name | default("etcd") }} ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/etcd.uuid diff --git a/roles/kubernetes-apps/ansible/defaults/main.yml b/roles/kubernetes-apps/ansible/defaults/main.yml index 923e9dc4a66d48fffa3647c308ec816817fe66de..14deb333dab58a275ec074995860faa0400715dc 100644 --- a/roles/kubernetes-apps/ansible/defaults/main.yml +++ b/roles/kubernetes-apps/ansible/defaults/main.yml @@ -51,18 +51,3 @@ netchecker_kubectl_memory_requests: 64M etcd_cert_dir: "/etc/ssl/etcd/ssl" calico_cert_dir: "/etc/calico/certs" canal_cert_dir: "/etc/canal/certs" - -# Linux capabilities to be dropped for k8s apps ran by container engines -apps_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap diff --git a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 index 3734aea96e4e9a112379fb22dcb6fbeda03e4939..06bb78b7c19d063010fcfde9cfad0852151a174c 100644 --- a/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/calico-policy-controller.yml.j2 @@ -25,12 +25,6 @@ spec: - name: calico-policy-controller image: {{ calico_policy_image_repo }}:{{ calico_policy_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ calico_policy_controller_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml index 7ea05c66bf6bb0a381b6c305952d2a1f088e27b2..41900ab33dce12de38b7249728b47b0014081615 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml @@ -23,12 +23,6 @@ spec: - name: REPORT_INTERVAL value: '{{ agent_report_interval }}' imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ netchecker_agent_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml index 9e246c56d7b04391bf1e80f190f60fdebd2fc78d..5a6a63f36eb30c72bbe8ee59ec3fe0aa65c2f4e8 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml @@ -24,12 +24,6 @@ spec: - name: REPORT_INTERVAL value: '{{ agent_report_interval }}' imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ netchecker_agent_cpu_limit }} diff --git a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml index a52c5d4a39708d3e26289055e4c101724bbb2662..c1d8ddb9f9e82fb78b2ead1c2781577fcc4fef5e 100644 --- a/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml +++ b/roles/kubernetes-apps/ansible/templates/netchecker-server-pod.yml @@ -33,9 +33,3 @@ spec: memory: {{ netchecker_kubectl_memory_requests }} args: - proxy - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml index 4e29a51fd8535a915b1325395934f3267384315a..874925adf58c3835755db123405c4dd8253abeb5 100644 --- a/roles/kubernetes/master/defaults/main.yml +++ b/roles/kubernetes/master/defaults/main.yml @@ -13,21 +13,6 @@ kube_apiserver_node_port_range: "30000-32767" etcd_config_dir: /etc/ssl/etcd etcd_cert_dir: "{{ etcd_config_dir }}/ssl" -# Linux capabilities to be dropped for k8s apps ran by container engines -apps_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap - # Limits for kube components kube_controller_memory_limit: 512M kube_controller_cpu_limit: 250m diff --git a/roles/kubernetes/master/tasks/main.yml b/roles/kubernetes/master/tasks/main.yml index b0bab4cdca2d3b88e410d76f40ccbcc8e960cd6f..a622594a182424202bd2e1be6640298ad2f5803f 100644 --- a/roles/kubernetes/master/tasks/main.yml +++ b/roles/kubernetes/master/tasks/main.yml @@ -2,9 +2,6 @@ - include: pre-upgrade.yml tags: k8s-pre-upgrade -- include: set_facts.yml - tags: facts - - name: Copy kubectl from hyperkube container command: "{{ docker_bin_dir }}/docker run --rm -v {{ bin_dir }}:/systembindir {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} /bin/cp /hyperkube /systembindir/kubectl" register: kube_task_result diff --git a/roles/kubernetes/master/tasks/set_facts.yml b/roles/kubernetes/master/tasks/set_facts.yml deleted file mode 100644 index d5c3250b32bf80cc3c78da8c31d993a3c86f57ec..0000000000000000000000000000000000000000 --- a/roles/kubernetes/master/tasks/set_facts.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Master | get kube user ID - shell: /usr/bin/id -u {{ kubelet_user }} || echo 0 - register: kube_uid - -- name: Master | get kube group ID - shell: /usr/bin/getent group {{ kubelet_group }} | cut -d':' -f3 || echo 0 - register: kube_gid - -- name: Master | get kube cert group ID - shell: /usr/bin/getent group {{ kube_cert_group }} | cut -d':' -f3 || echo 0 - register: kube_cert_gid - -- name: Master | get etcd cert group ID - shell: /usr/bin/getent group {{ etcd_cert_group }} | cut -d':' -f3 || echo 0 - register: etcd_cert_gid - -- set_fact: - kubelet_user_id: "{{ kube_uid.stdout }}" - kubelet_group_id: "{{ kube_gid.stdout }}" - kube_cert_group_id: "{{ kube_cert_gid.stdout }}" - etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 index cab219f60d4688222fbcf92d1c5c6819161200a5..c050306972821af4d26c9360b292afef1bf52905 100644 --- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 @@ -12,14 +12,6 @@ spec: - name: kube-apiserver image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} - add: - - DAC_OVERRIDE resources: limits: cpu: {{ kube_apiserver_cpu_limit }} diff --git a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 index 6db17f30af0456e5b6022655c0da6b76ca70597e..49dd05ba8089fd761da830fe7d91310fc9c148ab 100644 --- a/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2 @@ -11,17 +11,6 @@ spec: - name: kube-controller-manager image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - runAsUser: {{ kubelet_user_id }} - fsGroup: {{ kubelet_group_id }} - supplementalGroups: - - {{ kube_cert_group_id }} - - {{ etcd_cert_group_id }} - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ kube_controller_cpu_limit }} diff --git a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 index ded01dc79c410b9f4195121f96c48dbadc84a6bd..781e38d7b76229674111ba9a0991246267171c2f 100644 --- a/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 +++ b/roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2 @@ -11,17 +11,6 @@ spec: - name: kube-scheduler image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} imagePullPolicy: {{ k8s_image_pull_policy }} - securityContext: - runAsUser: {{ kubelet_user_id }} - fsGroup: {{ kubelet_group_id }} - supplementalGroups: - - {{ kube_cert_group_id }} - - {{ etcd_cert_group_id }} - capabilities: - drop: -{% for c in apps_drop_cap %} - - {{ c.upper() }} -{% endfor %} resources: limits: cpu: {{ kube_scheduler_cpu_limit }} diff --git a/roles/kubernetes/node/defaults/main.yml b/roles/kubernetes/node/defaults/main.yml index 777a751fad54cbf02106e70d498908ae9ce80b97..a74e52b77c158712f4417c666b4e6072145a9869 100644 --- a/roles/kubernetes/node/defaults/main.yml +++ b/roles/kubernetes/node/defaults/main.yml @@ -29,18 +29,3 @@ nginx_image_repo: nginx nginx_image_tag: 1.11.4-alpine etcd_config_dir: /etc/ssl/etcd - -# Linux capabilities to be dropped for container engines -apps_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - sys_chroot - - mknod - - audit_write - - setfcap diff --git a/roles/kubernetes/node/tasks/install.yml b/roles/kubernetes/node/tasks/install.yml index 52a32ccc3183b317dacb558a6cd3739e406e5b02..bfe4a8cc8e62ef46712de986634e17bd3deac4a4 100644 --- a/roles/kubernetes/node/tasks/install.yml +++ b/roles/kubernetes/node/tasks/install.yml @@ -26,6 +26,6 @@ notify: restart kubelet - name: install | Install kubelet launch script - template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner={{ kubelet_user }} mode=0755 backup=yes + template: src=kubelet-container.j2 dest="{{ bin_dir }}/kubelet" owner=kube mode=0755 backup=yes notify: restart kubelet when: kubelet_deployment_type == "docker" diff --git a/roles/kubernetes/node/tasks/main.yml b/roles/kubernetes/node/tasks/main.yml index 29e9ef4e02fd0bcea7439e0f3d36538d03edc6b0..3e0c095e18e822504ad9cd2cb3dcb67e4c84daad 100644 --- a/roles/kubernetes/node/tasks/main.yml +++ b/roles/kubernetes/node/tasks/main.yml @@ -4,9 +4,6 @@ {%- if inventory_hostname in groups['kube-master'] and inventory_hostname not in groups['kube-node'] -%}true{%- else -%}false{%- endif -%} tags: facts -- include: pre-upgrade.yml - tags: k8s-pre-upgrade - - include: install.yml tags: kubelet diff --git a/roles/kubernetes/node/tasks/pre-upgrade.yml b/roles/kubernetes/node/tasks/pre-upgrade.yml deleted file mode 100644 index 9a61170d7af20d9aaa20af7dbf261edb90046a54..0000000000000000000000000000000000000000 --- a/roles/kubernetes/node/tasks/pre-upgrade.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -- name: "Pre-upgrade | share access to kube certs for its users" - shell: chmod g+r {{ kube_cert_dir }}/*.pem - failed_when: false diff --git a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 index 2537d9ffdf4e953cb04b4d752512924974f75969..12ce01c75483596649d8fb474edf7103358de117 100644 --- a/roles/kubernetes/node/templates/kubelet.rkt.service.j2 +++ b/roles/kubernetes/node/templates/kubelet.rkt.service.j2 @@ -29,7 +29,7 @@ ExecStart=/usr/bin/rkt run \ --volume run,kind=host,source=/run,readOnly=false \ --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ --volume var-lib-docker,kind=host,source={{ docker_daemon_graph }},readOnly=false \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ + --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,readOnly=false \ --volume var-log,kind=host,source=/var/log \ --mount volume=dns,target=/etc/resolv.conf \ --mount volume=etc-cni,target=/etc/cni \ @@ -44,7 +44,6 @@ ExecStart=/usr/bin/rkt run \ --mount volume=var-log,target=/var/log \ --stage1-from-dir=stage1-fly.aci \ {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }} \ - --memory={{ kubelet_memory_limit }} --cpu={{ kubelet_cpu_limit }} \ --uuid-file-save=/var/run/kubelet.uuid \ --debug --exec=/kubelet -- \ $KUBE_LOGTOSTDERR \ diff --git a/roles/kubernetes/preinstall/meta/main.yml b/roles/kubernetes/preinstall/meta/main.yml index 34785ed88c6559db0466d8f05060bdb5c1f347a1..cf440f5e222c33d9670142d4309c243fa0610bac 100644 --- a/roles/kubernetes/preinstall/meta/main.yml +++ b/roles/kubernetes/preinstall/meta/main.yml @@ -2,7 +2,4 @@ dependencies: - role: adduser user: "{{ addusers.kube }}" - tags: [bootstrap-os, kubelet] - - role: adduser - user: "{{ addusers.netplug }}" - tags: [bootstrap-os, network] + tags: kubelet diff --git a/roles/kubernetes/preinstall/tasks/main.yml b/roles/kubernetes/preinstall/tasks/main.yml index 7a8760f958da9c1164d25326c9febc4c79e6b154..e7955ac76bb43b481d421d8e5e11b20f30a983e3 100644 --- a/roles/kubernetes/preinstall/tasks/main.yml +++ b/roles/kubernetes/preinstall/tasks/main.yml @@ -23,12 +23,6 @@ - include: set_facts.yml tags: facts -- include: set_resolv_facts.yml - tags: [bootstrap-os, resolvconf, facts] - -- include: set_uid_facts.yml - tags: [bootstrap-os, facts] - - name: gather os specific variables include_vars: "{{ item }}" with_first_found: @@ -48,7 +42,7 @@ file: path: "{{ kube_config_dir }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [kubelet, k8s-secrets, kube-controller-manager, kube-apiserver, bootstrap-os, apps, network, master, node] @@ -56,7 +50,7 @@ file: path: "{{ kube_script_dir }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [k8s-secrets, bootstrap-os] @@ -64,7 +58,7 @@ file: path: "{{ kube_manifest_dir }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube when: "{{ inventory_hostname in groups['k8s-cluster'] }}" tags: [kubelet, bootstrap-os, master, node] @@ -86,7 +80,7 @@ file: path: "{{ item }}" state: directory - owner: "{{ kubelet_user }}" + owner: kube with_items: - "/etc/cni/net.d" - "/opt/cni/bin" diff --git a/roles/kubernetes/preinstall/tasks/set_facts.yml b/roles/kubernetes/preinstall/tasks/set_facts.yml index d2fad6e3dd3a81208b46a1acfccfbd70fc9e23ae..456467a97cf1f9415e035af2280cdf1f7b25647f 100644 --- a/roles/kubernetes/preinstall/tasks/set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/set_facts.yml @@ -51,3 +51,6 @@ etcd_container_bin_dir: "{% if etcd_after_v3 %}/usr/local/bin/{% else %}/{% endif %}" - set_fact: peer_with_calico_rr: "{{ 'calico-rr' in groups and groups['calico-rr']|length > 0 }}" + +- include: set_resolv_facts.yml + tags: [bootstrap-os, resolvconf, facts] diff --git a/roles/kubernetes/preinstall/tasks/set_uid_facts.yml b/roles/kubernetes/preinstall/tasks/set_uid_facts.yml deleted file mode 100644 index 13a36b5dbd854911c52419487472e920b1d7bafc..0000000000000000000000000000000000000000 --- a/roles/kubernetes/preinstall/tasks/set_uid_facts.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -- name: Preinstall | get kube user ID - shell: /usr/bin/id -u {{ kubelet_user }} || echo 0 - register: kube_uid - -- name: Preinstall | get kube group ID - shell: /usr/bin/id -g {{ kubelet_group }} || echo 0 - register: kube_gid - -- name: Preinstall | get kube cert group ID - shell: /usr/bin/id -g {{ kube_cert_group }} || echo 0 - register: kube_cert_gid - -- name: Preinstall | get etcd cert group ID - shell: /usr/bin/id -g {{ etcd_cert_group }} || echo 0 - register: etcd_cert_gid - -- name: Preinstall | get netplug user ID - shell: /usr/bin/id -u {{ netplug_user }} || echo 0 - register: netplug_uid - -- name: Preinstall | get netplug group ID - shell: /usr/bin/getent group {{ netplug_group }} | cut -d':' -f3 || echo 0 - register: netplug_gid - -- set_fact: - kubelet_user_id: "{{ kube_uid.stdout }}" - kubelet_group_id: "{{ kube_gid.stdout }}" - kube_cert_group_id: "{{ kube_cert_gid.stdout }}" - etcd_cert_group_id: "{{ etcd_cert_gid.stdout }}" - netplug_user_id: "{{ netplug_uid.stdout }}" - netplug_group_id: "{{ netplug_gid.stdout }}" diff --git a/roles/kubernetes/secrets/files/make-ssl.sh b/roles/kubernetes/secrets/files/make-ssl.sh index cc0b49b95874d030f97788b138474adc8984d24c..dca9d9ab9d2f40f8ad026b90d2d4b65c8e2db137 100755 --- a/roles/kubernetes/secrets/files/make-ssl.sh +++ b/roles/kubernetes/secrets/files/make-ssl.sh @@ -101,8 +101,5 @@ if [ -n "$HOSTS" ]; then done fi -# Grant the group read access -chmod g+r *.pem - # Install certs mv *.pem ${SSLDIR}/ diff --git a/roles/kubernetes/secrets/tasks/gen_certs.yml b/roles/kubernetes/secrets/tasks/gen_certs.yml index d56e487e37810ca8ae419fadfdd97b558d4900c5..484afff635043741ac988f8d49017c4af5883769 100644 --- a/roles/kubernetes/secrets/tasks/gen_certs.yml +++ b/roles/kubernetes/secrets/tasks/gen_certs.yml @@ -140,11 +140,11 @@ file: path={{ kube_cert_dir }} group={{ kube_cert_group }} - owner={{ kubelet_user }} + owner=kube recurse=yes -- name: Gen_certs | set shared group permissions on keys - shell: chmod 0640 {{ kube_cert_dir}}/*.pem +- name: Gen_certs | set permissions on keys + shell: chmod 0600 {{ kube_cert_dir}}/*key.pem when: inventory_hostname in groups['kube-master'] changed_when: false diff --git a/roles/kubernetes/secrets/tasks/main.yml b/roles/kubernetes/secrets/tasks/main.yml index 81eb26743691b5f5555af275b0d964fe12bc9e16..4d25a94afb5b6ee34a6aa851d8195be449d39e8d 100644 --- a/roles/kubernetes/secrets/tasks/main.yml +++ b/roles/kubernetes/secrets/tasks/main.yml @@ -9,7 +9,6 @@ path={{ kube_cert_dir }} state=directory mode=o-rwx - owner={{ kubelet_user }} group={{ kube_cert_group }} - name: Make sure the tokens directory exits @@ -17,16 +16,14 @@ path={{ kube_token_dir }} state=directory mode=o-rwx - owner={{ kubelet_user }} - group={{ kubelet_group }} + group={{ kube_cert_group }} - name: Make sure the users directory exits file: path={{ kube_users_dir }} state=directory mode=o-rwx - owner={{ kubelet_user }} - group={{ kubelet_group }} + group={{ kube_cert_group }} - name: Populate users for basic auth in API lineinfile: diff --git a/roles/network_plugin/calico/defaults/main.yml b/roles/network_plugin/calico/defaults/main.yml index 6cd5c22137e1baea43fcc1d8509191115f7444b3..7681abc5c8fd648478fa58995500c014e74d01b0 100644 --- a/roles/network_plugin/calico/defaults/main.yml +++ b/roles/network_plugin/calico/defaults/main.yml @@ -20,23 +20,6 @@ global_as_num: "64512" # defaults. The value should be a number, not a string. # calico_mtu: 1500 -# Linux capabilities to be dropped for container engines -calico_drop_cap: - - chown - - dac_override - - fowner - - fsetid - - kill - - setgid - - setuid - - setpcap - - net_bind_service - - net_raw - - sys_chroot - - mknod - - audit_write - - setfcap - # Limits for apps calico_node_memory_limit: 500M calico_node_cpu_limit: 300m diff --git a/roles/network_plugin/calico/rr/tasks/main.yml b/roles/network_plugin/calico/rr/tasks/main.yml index 1cbbb43f85b7b51269dca9efa4a7243d904cf243..efe4616d25d8beb41fa23e86f177fc088e60bb46 100644 --- a/roles/network_plugin/calico/rr/tasks/main.yml +++ b/roles/network_plugin/calico/rr/tasks/main.yml @@ -12,8 +12,8 @@ dest: "{{ calico_cert_dir }}" state: directory mode: 0750 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Calico-rr | Link etcd certificates for calico-node file: @@ -31,8 +31,8 @@ path: /var/log/calico-rr state: directory mode: 0755 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Calico-rr | Write calico-rr.env for systemd init file template: src=calico-rr.env.j2 dest=/etc/calico/calico-rr.env diff --git a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 index 6d4b344d3572d1b384b2125699e5eaa766ed4e0e..f6da04a4d9560010b7b3f488ea6585d410c55a56 100644 --- a/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 +++ b/roles/network_plugin/calico/rr/templates/calico-rr.service.j2 @@ -6,7 +6,7 @@ Requires=docker.service [Service] EnvironmentFile=/etc/calico/calico-rr.env ExecStartPre=-{{ docker_bin_dir }}/docker rm -f calico-rr -ExecStart={{ docker_bin_dir }}/docker run --net=host \ +ExecStart={{ docker_bin_dir }}/docker run --net=host --privileged \ --name=calico-rr \ -e IP=${IP} \ -e IP6=${IP6} \ @@ -16,10 +16,6 @@ ExecStart={{ docker_bin_dir }}/docker run --net=host \ -e ETCD_KEY_FILE=${ETCD_KEY_FILE} \ -v /var/log/calico-rr:/var/log/calico \ -v {{ calico_cert_dir }}:{{ calico_cert_dir }}:ro \ -{% for c in calico_drop_cap %} - --cap-drop={{ c }} \ -{% endfor %} - -u {{ netplug_user_id }}:{{ netplug_group_id }} --group-add {{ etcd_cert_group }} \ --memory={{ calico_rr_memory_limit|regex_replace('Mi', 'M') }} --cpu-shares={{ calico_rr_cpu_limit|regex_replace('m', '') }} \ {{ calico_rr_image_repo }}:{{ calico_rr_image_tag }} diff --git a/roles/network_plugin/calico/tasks/main.yml b/roles/network_plugin/calico/tasks/main.yml index e88f7f5525a32f2d7d211141cc80b53739fc3cbf..dc9a96d192c1d9ee03febf5ae70d9c3ddbe3add6 100644 --- a/roles/network_plugin/calico/tasks/main.yml +++ b/roles/network_plugin/calico/tasks/main.yml @@ -9,16 +9,15 @@ template: src: "cni-calico.conf.j2" dest: "/etc/cni/net.d/10-calico.conf" - owner: "{{ kubelet_user }}" - group: "{{ kubelet_group }}" + owner: kube - name: Calico | Create calico certs directory file: dest: "{{ calico_cert_dir }}" state: directory mode: 0750 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Calico | Link etcd certificates for calico-node file: diff --git a/roles/network_plugin/canal/tasks/main.yml b/roles/network_plugin/canal/tasks/main.yml index a6dedeec4f258dc5cc6bef4cea71ca759aa93dca..7ccbcdf2ef0e2675a751eb57cbff55f15440f865 100644 --- a/roles/network_plugin/canal/tasks/main.yml +++ b/roles/network_plugin/canal/tasks/main.yml @@ -3,16 +3,15 @@ template: src: "cni-canal.conf.j2" dest: "/etc/cni/net.d/10-canal.conf" - owner: "{{ kubelet_user }}" - group: "{{ kubelet_group }}" + owner: kube - name: Canal | Create canal certs directory file: dest: "{{ canal_cert_dir }}" state: directory mode: 0750 - owner: "{{ netplug_user }}" - group: "{{ netplug_group }}" + owner: root + group: root - name: Canal | Link etcd certificates for canal-node file: