Support audit

inventory/sample/group_vars/k8s-cluster.yml

@@ -163,6 +163,9 @@ helm_deployment_type: host
 # K8s image pull policy (imagePullPolicy)
 k8s_image_pull_policy: IfNotPresent
 
+# audit log for kubernetes
+kubernetes_audit: false
+
 # Kubernetes dashboard
 # RBAC required. see docs/getting-started.md for access details.
 dashboard_enabled: true

roles/kubernetes/master/defaults/main.yml

@@ -24,6 +24,29 @@ kube_apiserver_storage_backend: etcd3
 # By default, force back to etcd2. Set to true to force etcd3 (experimental!)
 force_etcd3: false
 
+# audit support
+kubernetes_audit: false
+audit_log_path: /var/log/audit/kube-apiserver-audit.log
+# num days
+audit_log_maxage: 30
+# the num of audit logs to retain
+audit_log_maxbackups: 1
+ # the max size in MB to retain
+audit_log_maxsize: 100
+# policy file
+audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+
+# audit log hostpath
+audit_log_name: audit-logs
+audit_log_hostpath: /var/log/kubernetes/audit
+audit_log_mountpath: /var/log/audit
+audit_log_writable: true
+
+# audit policy hostpath
+audit_policy_name: audit-policy
+audit_policy_hostpath: /etc/kubernetes/audit-policy
+audit_policy_mountpath: "{{ audit_policy_hostpath }}"
+
 # Limits for kube components
 kube_controller_memory_limit: 512M
 kube_controller_cpu_limit: 250m

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -65,6 +65,16 @@
   command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
   changed_when: false
 
+- name: Create audit-policy directory
+  file: path={{ kube_config_dir }}/audit-policy state=directory
+  when: kubernetes_audit|default(false)
+
+- name: Write api audit policy yaml
+  template:
+    src: apiserver-audit-policy.yaml.j2
+    dest: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+  when: kubernetes_audit|default(false)
+
 - name: gets the kubeadm version
   command: "{{ bin_dir }}/kubeadm version -o short"
   register: kubeadm_output

roles/kubernetes/master/tasks/static-pod-setup.yml

 ---
+- name: Create audit-policy directory
+  file: path={{ kube_config_dir }}/audit-policy state=directory
+  tags:
+    - kube-apiserver
+  when: kubernetes_audit|default(false)
+
+- name: Write api audit policy yaml
+  template:
+    src: apiserver-audit-policy.yaml.j2
+    dest: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+  notify: Master | Restart apiserver
+  tags:
+    - kube-apiserver
+  when: kubernetes_audit|default(false)
+
 - name: Write kube-apiserver manifest
   template:
     src: manifests/kube-apiserver.manifest.j2

roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2

+apiVersion: audit.k8s.io/v1beta1
+kind: Policy
+rules:
+  # The following requests were manually identified as high-volume and low-risk,
+  # so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
+  - level: None
+    # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
+    # TODO(#46983): Change this to the ingress controller service account.
+    users: ["system:unsecured"]
+    namespaces: ["kube-system"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["configmaps"]
+  - level: None
+    users: ["kubelet"] # legacy kubelet identity
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    userGroups: ["system:nodes"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    users:
+      - system:kube-controller-manager
+      - system:kube-scheduler
+      - system:serviceaccount:kube-system:endpoint-controller
+    verbs: ["get", "update"]
+    namespaces: ["kube-system"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints"]
+  - level: None
+    users: ["system:apiserver"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+  # Don't log HPA fetching metrics.
+  - level: None
+    users:
+      - system:kube-controller-manager
+    verbs: ["get", "list"]
+    resources:
+      - group: "metrics.k8s.io"
+  # Don't log these read-only URLs.
+  - level: None
+    nonResourceURLs:
+      - /healthz*
+      - /version
+      - /swagger*
+  # Don't log events requests.
+  - level: None
+    resources:
+      - group: "" # core
+        resources: ["events"]
+  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
+  # so only log at the Metadata level.
+  - level: Metadata
+    resources:
+      - group: "" # core
+        resources: ["secrets", "configmaps"]
+      - group: authentication.k8s.io
+        resources: ["tokenreviews"]
+    omitStages:
+      - "RequestReceived"
+  # Get responses can be large; skip them.
+  - level: Request
+    verbs: ["get", "list", "watch"]
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for known APIs
+  - level: RequestResponse
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for all other requests.
+  - level: Metadata
+    omitStages:
+      - "RequestReceived"

roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2

@@ -12,6 +12,12 @@ etcd:
       caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
       certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
       keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+{% if kubernetes_audit %}
+auditPolicy:
+  logDir: {{ audit_log_path }}
+  logMaxAge: {{ audit_log_maxage }}
+  path: {{ audit_policy_file }}
+{% endif %}
 networking:
   dnsDomain: {{ dns_domain }}
   serviceSubnet: {{ kube_service_addresses }}
@@ -82,6 +88,12 @@ controllerManagerExtraArgs:
   node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
   node-monitor-period: {{ kube_controller_node_monitor_period }}
   pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kubernetes_audit %}
+apiServerExtraVolumes:
+- name: {{ audit_policy_name }}
+  hostPath: {{ audit_policy_hostpath }}
+  mountPath: {{ audit_policy_mountpath }}
+{% endif %}
 {% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
 controllerManagerExtraVolumes:
 - name: openstackcacert
@@ -113,3 +125,7 @@ nodeRegistration:
   taints:
   - effect: NoSchedule
     key: node-role.kubernetes.io/master
+{% if kubernetes_audit %}
+featureGates:
+  Auditing: true
+{% endif %}

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

@@ -28,6 +28,13 @@ spec:
     command:
     - /hyperkube
     - apiserver
+{% if kubernetes_audit %}
+    - --audit-log-path={{ audit_log_path }}
+    - --audit-log-maxage={{ audit_log_maxage }}
+    - --audit-log-maxbackup={{ audit_log_maxbackups }}
+    - --audit-log-maxsize={{ audit_log_maxsize }}
+    - --audit-policy-file={{ audit_policy_file }} 
+{% endif %}
     - --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
     - --etcd-servers={{ etcd_access_addresses }}
 {%   if etcd_events_cluster_enabled %}
@@ -184,6 +191,14 @@ spec:
     - mountPath: /etc/ssl/certs/ca-bundle.crt
       name: rhel-ca-bundle
       readOnly: true
+{% endif %}
+{% if kubernetes_audit %}
+    - mountPath: {{ audit_log_mountpath }}
+      name: {{ audit_log_name }}
+      Writable: true
+    - mountPath: {{ audit_policy_mountpath }}
+      name: {{ audit_policy_name }}
+      Writable: true
 {% endif %}
   volumes:
   - hostPath:
@@ -205,3 +220,11 @@ spec:
       path: /etc/ssl/certs/ca-bundle.crt
     name: rhel-ca-bundle
 {% endif %}
+{% if kubernetes_audit %}
+  - hostPath:
+      path: {{ audit_log_hostpath }}
+    name: {{ audit_log_name }}
+  - hostPath:
+      path: {{ audit_policy_hostpath }}
+    name: {{ audit_policy_name }}
+{% endif %}

tests/files/gce_centos-weave-kubeadm.yml

@@ -9,5 +9,6 @@ startup_script: ""
 kube_network_plugin: weave
 kubeadm_enabled: true
 deploy_netchecker: true
+kubernetes_audit: true
 kubedns_min_replicas: 1
 cloud_provider: gce

tests/files/gce_centos7-flannel-addons.yml

@@ -8,6 +8,7 @@ mode: ha
 kube_network_plugin: flannel
 helm_enabled: true
 efk_enabled: true
+kubernetes_audit: true
 etcd_events_cluster_setup: true
 local_volume_provisioner_enabled: true
 etcd_deployment_type: host