Merge pull request #3102 from mirwan/psp

PodSecurityPolicy admission controller support

inventory/sample/group_vars/k8s-cluster.yml

@@ -172,6 +172,9 @@ k8s_image_pull_policy: IfNotPresent
 # audit log for kubernetes
 kubernetes_audit: false
 
+# pod security policy (RBAC must be enabled either by having 'RBAC' in authorization_modes or kubeadm enabled)
+podsecuritypolicy_enabled: false
+
 # Kubernetes dashboard
 # RBAC required. see docs/getting-started.md for access details.
 dashboard_enabled: true

roles/kubernetes-apps/ansible/defaults/main.yml

@@ -42,6 +42,12 @@ netchecker_server_memory_limit: 256M
 netchecker_server_cpu_requests: 50m
 netchecker_server_memory_requests: 64M
 
+# SecurityContext when PodSecurityPolicy is enabled
+netchecker_agent_user: 1000
+netchecker_server_user: 1000
+netchecker_agent_group: 1000
+netchecker_server_group: 1000
+
 # Dashboard
 dashboard_enabled: true
 dashboard_image_repo: gcr.io/google_containers/kubernetes-dashboard-{{ image_arch }}

roles/kubernetes-apps/ansible/tasks/netchecker.yml

@@ -20,18 +20,32 @@
   tags:
     - upgrade
 
+- name: Kubernetes Apps | Netchecker Templates list
+  set_fact:
+    netchecker_templates:
+      - {file: netchecker-agent-sa.yml, type: sa, name: netchecker-agent}
+      - {file: netchecker-agent-ds.yml, type: ds, name: netchecker-agent}
+      - {file: netchecker-agent-hostnet-ds.yml, type: ds, name: netchecker-agent-hostnet}
+      - {file: netchecker-server-sa.yml, type: sa, name: netchecker-server}
+      - {file: netchecker-server-clusterrole.yml, type: clusterrole, name: netchecker-server}
+      - {file: netchecker-server-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-server}
+      - {file: netchecker-server-deployment.yml, type: deployment, name: netchecker-server}
+      - {file: netchecker-server-svc.yml, type: svc, name: netchecker-service}
+    netchecker_templates_for_psp:
+      - {file: netchecker-agent-hostnet-psp.yml, type: podsecuritypolicy, name: netchecker-agent-hostnet-policy}
+      - {file: netchecker-agent-hostnet-clusterrole.yml, type: clusterrole, name: netchecker-agent}
+      - {file: netchecker-agent-hostnet-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-agent}
+
+- name: Kubernetes Apps | Append extra templates to Netchecker Templates list for PodSecurityPolicy
+  set_fact:
+    netchecker_templates: "{{ netchecker_templates_for_psp + netchecker_templates}}"
+  when: podsecuritypolicy_enabled
+
 - name: Kubernetes Apps | Lay Down Netchecker Template
   template:
     src: "{{item.file}}.j2"
     dest: "{{kube_config_dir}}/{{item.file}}"
-  with_items:
-    - {file: netchecker-agent-ds.yml, type: ds, name: netchecker-agent}
-    - {file: netchecker-agent-hostnet-ds.yml, type: ds, name: netchecker-agent-hostnet}
-    - {file: netchecker-server-sa.yml, type: sa, name: netchecker-server}
-    - {file: netchecker-server-clusterrole.yml, type: clusterrole, name: netchecker-server}
-    - {file: netchecker-server-clusterrolebinding.yml, type: clusterrolebinding, name: netchecker-server}
-    - {file: netchecker-server-deployment.yml, type: deployment, name: netchecker-server}
-    - {file: netchecker-server-svc.yml, type: svc, name: netchecker-service}
+  with_items: "{{ netchecker_templates }}"
   register: manifests
   when:
     - inventory_hostname == groups['kube-master'][0]

roles/kubernetes-apps/ansible/templates/netchecker-agent-ds.yml.j2

@@ -40,6 +40,10 @@ spec:
             requests:
               cpu: {{ netchecker_agent_cpu_requests }}
               memory: {{ netchecker_agent_memory_requests }}
+          securityContext:
+            runAsUser: {{ netchecker_agent_user | default('0') }}
+            runAsGroup: {{ netchecker_agent_group | default('0') }}
+      serviceAccountName: netchecker-agent
   updateStrategy:
     rollingUpdate:
       maxUnavailable: 100%

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrole.yml.j2

+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: psp:netchecker-agent-hostnet
+  namespace: {{ netcheck_namespace }}
+rules:
+  - apiGroups:
+    - policy
+    resourceNames:
+    - netchecker-agent-hostnet
+    resources:
+    - podsecuritypolicies
+    verbs:
+    - use

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-clusterrolebinding.yml.j2

+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: psp:netchecker-agent-hostnet
+  namespace: {{ netcheck_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: netchecker-agent-hostnet
+    namespace: {{ netcheck_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: psp:netchecker-agent-hostnet
+  apiGroup: rbac.authorization.k8s.io

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-ds.yml.j2

@@ -44,6 +44,10 @@ spec:
             requests:
               cpu: {{ netchecker_agent_cpu_requests }}
               memory: {{ netchecker_agent_memory_requests }}
+          securityContext:
+            runAsUser: {{ netchecker_agent_user | default('0') }}
+            runAsGroup: {{ netchecker_agent_group | default('0') }}
+      serviceAccountName: netchecker-agent
   updateStrategy:
     rollingUpdate:
       maxUnavailable: 100%

roles/kubernetes-apps/ansible/templates/netchecker-agent-hostnet-psp.yml.j2

+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: netchecker-agent-hostnet
+  annotations:
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+{% if apparmor_enabled %}
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+{% endif %}
+  labels:
+    kubernetes.io/cluster-service: 'true'
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: true
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'MustRunAsNonRoot'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false

roles/kubernetes-apps/ansible/templates/netchecker-agent-sa.yml.j2

+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netchecker-agent
+  namespace: {{ netcheck_namespace }}
+  labels:
+    kubernetes.io/cluster-service: "true"

roles/kubernetes-apps/ansible/templates/netchecker-server-deployment.yml.j2

@@ -23,6 +23,9 @@ spec:
             requests:
               cpu: {{ netchecker_server_cpu_requests }}
               memory: {{ netchecker_server_memory_requests }}
+          securityContext:
+            runAsUser: {{ netchecker_server_user | default('0') }}
+            runAsGroup: {{ netchecker_server_group | default('0') }}
           ports:
             - containerPort: 8081
           args:

roles/kubernetes-apps/cluster_roles/tasks/main.yml

@@ -11,6 +11,46 @@
   delay: 6
   when: inventory_hostname == groups['kube-master'][0]
 
+- name: Kubernetes Apps | Check AppArmor status
+  command: which apparmor_parser
+  register: apparmor_status
+  when:
+    - podsecuritypolicy_enabled
+    - inventory_hostname == groups['kube-master'][0]
+  failed_when: false
+
+- name: Kubernetes Apps | Set apparmor_enabled
+  set_fact:
+    apparmor_enabled: "{{ apparmor_status.rc == 0 }}"
+  when:
+    - podsecuritypolicy_enabled
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Apps | Render templates for PodSecurityPolicy
+  template:
+    src: "{{ item.file }}.j2"
+    dest: "{{ kube_config_dir }}/{{ item.file }}"
+  register: psp_manifests
+  with_items:
+    - {file: psp.yml, type: psp, name: psp}
+    - {file: psp-cr.yml, type: clusterrole, name: psp-cr}
+    - {file: psp-crb.yml, type: rolebinding, name: psp-crb}
+  when:
+    - podsecuritypolicy_enabled
+    - inventory_hostname == groups['kube-master'][0]
+
+- name: Kubernetes Apps | Add policies, roles, bindings for PodSecurityPolicy
+  kube:
+    name: "{{item.item.name}}"
+    kubectl: "{{bin_dir}}/kubectl"
+    resource: "{{item.item.type}}"
+    filename: "{{kube_config_dir}}/{{item.item.file}}"
+    state: "latest"
+  with_items: "{{ psp_manifests.results }}"
+  when:
+    - inventory_hostname == groups['kube-master'][0]
+    - not item|skipped
+
 - name: Kubernetes Apps | Add ClusterRoleBinding to admit nodes
   template:
     src: "node-crb.yml.j2"

roles/kubernetes-apps/cluster_roles/templates/psp-cr.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: psp:privileged
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - policy
+  resourceNames:
+  - privileged
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: psp:restricted
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+rules:
+- apiGroups:
+  - policy
+  resourceNames:
+  - restricted
+  resources:
+  - podsecuritypolicies
+  verbs:
+  - use

roles/kubernetes-apps/cluster_roles/templates/psp-crb.yml.j2

+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: psp:any:restricted
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:restricted
+subjects:
+- kind: Group
+  name: system:authenticated
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: psp:kube-system:privileged
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:privileged
+subjects:
+- kind: Group
+  name: system:masters
+  apiGroup: rbac.authorization.k8s.io
+- kind: Group
+  name: system:serviceaccounts:kube-system
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: psp:nodes:privileged
+  namespace: kube-system
+  annotations:
+    kubernetes.io/description: 'Allow nodes to create privileged pods. Should
+      be used in combination with the NodeRestriction admission plugin to limit
+      nodes to mirror pods bound to themselves.'
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/cluster-service: 'true'
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: psp:privileged
+subjects:
+  - kind: Group
+    apiGroup: rbac.authorization.k8s.io
+    name: system:nodes
+  - kind: User
+    apiGroup: rbac.authorization.k8s.io
+    # Legacy node ID
+    name: kubelet

roles/kubernetes-apps/cluster_roles/templates/psp.yml.j2

+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: restricted
+  annotations:
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+{% if apparmor_enabled %}
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+{% endif %}
+  labels:
+    kubernetes.io/cluster-service: 'true'
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'MustRunAsNonRoot'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: privileged
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: true
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+  - '*'
+  volumes:
+  - '*'
+  hostNetwork: true
+  hostPorts:
+  - min: 0
+    max: 65535
+  hostIPC: true
+  hostPID: true
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'RunAsAny'
+  fsGroup:
+    rule: 'RunAsAny'
+  readOnlyRootFilesystem: false

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/tasks/main.yml

@@ -37,20 +37,33 @@
   when:
     - inventory_hostname == groups['kube-master'][0]
 
+- name: CephFS Provisioner | Templates list
+  set_fact:
+    cephfs_provisioner_templates:
+      - { name: 00-namespace, file: 00-namespace.yml, type: ns }
+      - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret }
+      - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa }
+      - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole }
+      - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding }
+      - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role }
+      - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding }
+      - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy }
+      - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc }
+    cephfs_provisioner_templates_for_psp:
+      - { name: psp-cephfs-provisioner, file: psp-cephfs-provisioner.yml, type: psp }
+
+- name: CephFS Provisioner | Append extra templates to CephFS Provisioner Templates list for PodSecurityPolicy
+  set_fact:
+    cephfs_provisioner_templates: "{{ cephfs_provisioner_templates_for_psp + cephfs_provisioner_templates }}"
+  when:
+    - podsecuritypolicy_enabled
+    - cephfs_provisioner_namespace != "kube-system"
+
 - name: CephFS Provisioner | Create manifests
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/cephfs_provisioner/{{ item.file }}"
-  with_items:
-    - { name: 00-namespace, file: 00-namespace.yml, type: ns }
-    - { name: secret-cephfs-provisioner, file: secret-cephfs-provisioner.yml, type: secret }
-    - { name: sa-cephfs-provisioner, file: sa-cephfs-provisioner.yml, type: sa }
-    - { name: clusterrole-cephfs-provisioner, file: clusterrole-cephfs-provisioner.yml, type: clusterrole }
-    - { name: clusterrolebinding-cephfs-provisioner, file: clusterrolebinding-cephfs-provisioner.yml, type: clusterrolebinding }
-    - { name: role-cephfs-provisioner, file: role-cephfs-provisioner.yml, type: role }
-    - { name: rolebinding-cephfs-provisioner, file: rolebinding-cephfs-provisioner.yml, type: rolebinding }
-    - { name: deploy-cephfs-provisioner, file: deploy-cephfs-provisioner.yml, type: deploy }
-    - { name: sc-cephfs-provisioner, file: sc-cephfs-provisioner.yml, type: sc }
+  with_items: "{{ cephfs_provisioner_templates }}"
   register: cephfs_provisioner_manifests
   when: inventory_hostname == groups['kube-master'][0]
 

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/clusterrole-cephfs-provisioner.yml.j2

@@ -23,3 +23,11 @@ rules:
   - apiGroups: [""]
     resources: ["secrets"]
     verbs: ["get", "create", "delete"]
+  - apiGroups:
+    - policy
+    resourceNames:
+    - cephfs-provisioner
+    resources:
+    - podsecuritypolicies
+    verbs:
+    - use

roles/kubernetes-apps/external_provisioner/cephfs_provisioner/templates/psp-cephfs-provisioner.yml.j2

+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: cephfs-provisioner
+  annotations:
+    seccomp.security.alpha.kubernetes.io/defaultProfileName:  'docker/default'
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default'
+{% if apparmor_enabled %}
+    apparmor.security.beta.kubernetes.io/defaultProfileName:  'runtime/default'
+    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
+{% endif %}
+  labels:
+    kubernetes.io/cluster-service: 'true'
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  requiredDropCapabilities:
+    - ALL
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'persistentVolumeClaim'
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  runAsUser:
+    rule: 'RunAsAny'
+  seLinux:
+    rule: 'RunAsAny'
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      - min: 1
+        max: 65535
+  readOnlyRootFilesystem: false

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/tasks/main.yml

@@ -19,17 +19,32 @@
     group: root
     mode: 0755
 
+- name: Local Volume Provisioner | Templates list
+  set_fact:
+    local_volume_provisioner_templates:
+      - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns }
+      - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa }
+      - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type: clusterrolebinding }
+      - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type: cm }
+      - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type: ds }
+      - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type: sc }
+    local_volume_provisioner_templates_for_psp_not_system_ns:
+      - { name: local-volume-provisioner-psp, file: local-volume-provisioner-psp.yml, type: psp }
+      - { name: local-volume-provisioner-psp-role, file: local-volume-provisioner-psp-role.yml, type: role }
+      - { name: local-volume-provisioner-psp-rb, file: local-volume-provisioner-psp-rb.yml, type: rolebinding }
+
+- name: Local Volume Provisioner | Insert extra templates to Local Volume Provisioner templates list for PodSecurityPolicy
+  set_fact:
+    local_volume_provisioner_templates: "{{ local_volume_provisioner_templates[:2] + local_volume_provisioner_templates_for_psp_not_system_ns + local_volume_provisioner_templates[3:] }}"
+  when:
+    - podsecuritypolicy_enabled
+    - local_volume_provisioner_namespace != "kube-system"
+
 - name: Local Volume Provisioner | Create manifests
   template:
     src: "{{ item.file }}.j2"
     dest: "{{ kube_config_dir }}/addons/local_volume_provisioner/{{ item.file }}"
-  with_items:
-    - { name: local-volume-provisioner-ns, file: local-volume-provisioner-ns.yml, type: ns }
-    - { name: local-volume-provisioner-sa, file: local-volume-provisioner-sa.yml, type: sa }
-    - { name: local-volume-provisioner-clusterrolebinding, file: local-volume-provisioner-clusterrolebinding.yml, type, clusterrolebinding }
-    - { name: local-volume-provisioner-cm, file: local-volume-provisioner-cm.yml, type, cm }
-    - { name: local-volume-provisioner-ds, file: local-volume-provisioner-ds.yml, type, ds }
-    - { name: local-volume-provisioner-sc, file: local-volume-provisioner-sc.yml, type, sc }
+  with_items: "{{ local_volume_provisioner_templates }}"
   register: local_volume_provisioner_manifests
   when: inventory_hostname == groups['kube-master'][0]
 

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp-cr.yml.j2

+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: psp:local-volume-provisioner
+  namespace: {{ local_volume_provisioner_namespace }}
+rules:
+  - apiGroups:
+    - policy
+    resourceNames:
+    - local-volume-provisioner
+    resources:
+    - podsecuritypolicies
+    verbs:
+    - use

roles/kubernetes-apps/external_provisioner/local_volume_provisioner/templates/local-volume-provisioner-psp-rb.yml.j2

+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: psp:local-volume-provisioner
+  namespace: {{ local_volume_provisioner_namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: local-volume-provisioner
+    namespace: {{ local_volume_provisioner_namespace }}
+roleRef:
+  kind: ClusterRole
+  name: psp:local-volume-provisioner
+  apiGroup: rbac.authorization.k8s.io