From 2ffc1afe400f80083c22cee76d138fecfd628c49 Mon Sep 17 00:00:00 2001
From: rongzhang <rongzhang@alauda.io>
Date: Wed, 15 Aug 2018 16:41:13 +0800
Subject: [PATCH] Support audit
---
inventory/sample/group_vars/k8s-cluster.yml | 3 +
roles/kubernetes/master/defaults/main.yml | 23 ++++
.../kubernetes/master/tasks/kubeadm-setup.yml | 10 ++
.../master/tasks/static-pod-setup.yml | 15 +++
.../templates/apiserver-audit-policy.yaml.j2 | 125 ++++++++++++++++++
.../templates/kubeadm-config.v1alpha2.yaml.j2 | 16 +++
.../manifests/kube-apiserver.manifest.j2 | 23 ++++
tests/files/gce_centos-weave-kubeadm.yml | 1 +
tests/files/gce_centos7-flannel-addons.yml | 1 +
9 files changed, 217 insertions(+)
create mode 100644 roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2
diff --git a/inventory/sample/group_vars/k8s-cluster.yml b/inventory/sample/group_vars/k8s-cluster.yml
index 795d76eb3..3e43b0833 100644
--- a/inventory/sample/group_vars/k8s-cluster.yml
+++ b/inventory/sample/group_vars/k8s-cluster.yml
@@ -163,6 +163,9 @@ helm_deployment_type: host
# K8s image pull policy (imagePullPolicy)
k8s_image_pull_policy: IfNotPresent
+# audit log for kubernetes
+kubernetes_audit: false
+
# Kubernetes dashboard
# RBAC required. see docs/getting-started.md for access details.
dashboard_enabled: true
diff --git a/roles/kubernetes/master/defaults/main.yml b/roles/kubernetes/master/defaults/main.yml
index 82669e8b3..68a09cef0 100644
--- a/roles/kubernetes/master/defaults/main.yml
+++ b/roles/kubernetes/master/defaults/main.yml
@@ -24,6 +24,29 @@ kube_apiserver_storage_backend: etcd3
# By default, force back to etcd2. Set to true to force etcd3 (experimental!)
force_etcd3: false
+# audit support
+kubernetes_audit: false
+audit_log_path: /var/log/audit/kube-apiserver-audit.log
+# num days
+audit_log_maxage: 30
+# the num of audit logs to retain
+audit_log_maxbackups: 1
+ # the max size in MB to retain
+audit_log_maxsize: 100
+# policy file
+audit_policy_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+
+# audit log hostpath
+audit_log_name: audit-logs
+audit_log_hostpath: /var/log/kubernetes/audit
+audit_log_mountpath: /var/log/audit
+audit_log_writable: true
+
+# audit policy hostpath
+audit_policy_name: audit-policy
+audit_policy_hostpath: /etc/kubernetes/audit-policy
+audit_policy_mountpath: "{{ audit_policy_hostpath }}"
+
# Limits for kube components
kube_controller_memory_limit: 512M
kube_controller_cpu_limit: 250m
diff --git a/roles/kubernetes/master/tasks/kubeadm-setup.yml b/roles/kubernetes/master/tasks/kubeadm-setup.yml
index 80e89acc9..2ba7485a1 100644
--- a/roles/kubernetes/master/tasks/kubeadm-setup.yml
+++ b/roles/kubernetes/master/tasks/kubeadm-setup.yml
@@ -65,6 +65,16 @@
command: "cp -TR {{ etcd_cert_dir }} {{ kube_config_dir }}/ssl/etcd"
changed_when: false
+- name: Create audit-policy directory
+ file: path={{ kube_config_dir }}/audit-policy state=directory
+ when: kubernetes_audit|default(false)
+
+- name: Write api audit policy yaml
+ template:
+ src: apiserver-audit-policy.yaml.j2
+ dest: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+ when: kubernetes_audit|default(false)
+
- name: gets the kubeadm version
command: "{{ bin_dir }}/kubeadm version -o short"
register: kubeadm_output
diff --git a/roles/kubernetes/master/tasks/static-pod-setup.yml b/roles/kubernetes/master/tasks/static-pod-setup.yml
index ca00ca33c..b1fbdc095 100644
--- a/roles/kubernetes/master/tasks/static-pod-setup.yml
+++ b/roles/kubernetes/master/tasks/static-pod-setup.yml
@@ -1,4 +1,19 @@
---
+- name: Create audit-policy directory
+ file: path={{ kube_config_dir }}/audit-policy state=directory
+ tags:
+ - kube-apiserver
+ when: kubernetes_audit|default(false)
+
+- name: Write api audit policy yaml
+ template:
+ src: apiserver-audit-policy.yaml.j2
+ dest: "{{ kube_config_dir }}/audit-policy/apiserver-audit-policy.yaml"
+ notify: Master | Restart apiserver
+ tags:
+ - kube-apiserver
+ when: kubernetes_audit|default(false)
+
- name: Write kube-apiserver manifest
template:
src: manifests/kube-apiserver.manifest.j2
diff --git a/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2 b/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2
new file mode 100644
index 000000000..40d6a8bb5
--- /dev/null
+++ b/roles/kubernetes/master/templates/apiserver-audit-policy.yaml.j2
@@ -0,0 +1,125 @@
+apiVersion: audit.k8s.io/v1beta1
+kind: Policy
+rules:
+ # The following requests were manually identified as high-volume and low-risk,
+ # so drop them.
+ - level: None
+ users: ["system:kube-proxy"]
+ verbs: ["watch"]
+ resources:
+ - group: "" # core
+ resources: ["endpoints", "services", "services/status"]
+ - level: None
+ # Ingress controller reads `configmaps/ingress-uid` through the unsecured port.
+ # TODO(#46983): Change this to the ingress controller service account.
+ users: ["system:unsecured"]
+ namespaces: ["kube-system"]
+ verbs: ["get"]
+ resources:
+ - group: "" # core
+ resources: ["configmaps"]
+ - level: None
+ users: ["kubelet"] # legacy kubelet identity
+ verbs: ["get"]
+ resources:
+ - group: "" # core
+ resources: ["nodes", "nodes/status"]
+ - level: None
+ userGroups: ["system:nodes"]
+ verbs: ["get"]
+ resources:
+ - group: "" # core
+ resources: ["nodes", "nodes/status"]
+ - level: None
+ users:
+ - system:kube-controller-manager
+ - system:kube-scheduler
+ - system:serviceaccount:kube-system:endpoint-controller
+ verbs: ["get", "update"]
+ namespaces: ["kube-system"]
+ resources:
+ - group: "" # core
+ resources: ["endpoints"]
+ - level: None
+ users: ["system:apiserver"]
+ verbs: ["get"]
+ resources:
+ - group: "" # core
+ resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+ # Don't log HPA fetching metrics.
+ - level: None
+ users:
+ - system:kube-controller-manager
+ verbs: ["get", "list"]
+ resources:
+ - group: "metrics.k8s.io"
+ # Don't log these read-only URLs.
+ - level: None
+ nonResourceURLs:
+ - /healthz*
+ - /version
+ - /swagger*
+ # Don't log events requests.
+ - level: None
+ resources:
+ - group: "" # core
+ resources: ["events"]
+ # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
+ # so only log at the Metadata level.
+ - level: Metadata
+ resources:
+ - group: "" # core
+ resources: ["secrets", "configmaps"]
+ - group: authentication.k8s.io
+ resources: ["tokenreviews"]
+ omitStages:
+ - "RequestReceived"
+ # Get responses can be large; skip them.
+ - level: Request
+ verbs: ["get", "list", "watch"]
+ resources:
+ - group: "" # core
+ - group: "admissionregistration.k8s.io"
+ - group: "apiextensions.k8s.io"
+ - group: "apiregistration.k8s.io"
+ - group: "apps"
+ - group: "authentication.k8s.io"
+ - group: "authorization.k8s.io"
+ - group: "autoscaling"
+ - group: "batch"
+ - group: "certificates.k8s.io"
+ - group: "extensions"
+ - group: "metrics.k8s.io"
+ - group: "networking.k8s.io"
+ - group: "policy"
+ - group: "rbac.authorization.k8s.io"
+ - group: "settings.k8s.io"
+ - group: "storage.k8s.io"
+ omitStages:
+ - "RequestReceived"
+ # Default level for known APIs
+ - level: RequestResponse
+ resources:
+ - group: "" # core
+ - group: "admissionregistration.k8s.io"
+ - group: "apiextensions.k8s.io"
+ - group: "apiregistration.k8s.io"
+ - group: "apps"
+ - group: "authentication.k8s.io"
+ - group: "authorization.k8s.io"
+ - group: "autoscaling"
+ - group: "batch"
+ - group: "certificates.k8s.io"
+ - group: "extensions"
+ - group: "metrics.k8s.io"
+ - group: "networking.k8s.io"
+ - group: "policy"
+ - group: "rbac.authorization.k8s.io"
+ - group: "settings.k8s.io"
+ - group: "storage.k8s.io"
+ omitStages:
+ - "RequestReceived"
+ # Default level for all other requests.
+ - level: Metadata
+ omitStages:
+ - "RequestReceived"
diff --git a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2 b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
index 907ea55f2..29aac6f87 100644
--- a/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
+++ b/roles/kubernetes/master/templates/kubeadm-config.v1alpha2.yaml.j2
@@ -12,6 +12,12 @@ etcd:
caFile: {{ kube_config_dir }}/ssl/etcd/ca.pem
certFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}.pem
keyFile: {{ kube_config_dir }}/ssl/etcd/node-{{ inventory_hostname }}-key.pem
+{% if kubernetes_audit %}
+auditPolicy:
+ logDir: {{ audit_log_path }}
+ logMaxAge: {{ audit_log_maxage }}
+ path: {{ audit_policy_file }}
+{% endif %}
networking:
dnsDomain: {{ dns_domain }}
serviceSubnet: {{ kube_service_addresses }}
@@ -82,6 +88,12 @@ controllerManagerExtraArgs:
node-monitor-grace-period: {{ kube_controller_node_monitor_grace_period }}
node-monitor-period: {{ kube_controller_node_monitor_period }}
pod-eviction-timeout: {{ kube_controller_pod_eviction_timeout }}
+{% if kubernetes_audit %}
+apiServerExtraVolumes:
+- name: {{ audit_policy_name }}
+ hostPath: {{ audit_policy_hostpath }}
+ mountPath: {{ audit_policy_mountpath }}
+{% endif %}
{% if cloud_provider is defined and cloud_provider in ["openstack"] and openstack_cacert is defined %}
controllerManagerExtraVolumes:
- name: openstackcacert
@@ -113,3 +125,7 @@ nodeRegistration:
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/master
+{% if kubernetes_audit %}
+featureGates:
+ Auditing: true
+{% endif %}
diff --git a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2 b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
index 82bd1db93..9cec5ded7 100644
--- a/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
+++ b/roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2
@@ -28,6 +28,13 @@ spec:
command:
- /hyperkube
- apiserver
+{% if kubernetes_audit %}
+ - --audit-log-path={{ audit_log_path }}
+ - --audit-log-maxage={{ audit_log_maxage }}
+ - --audit-log-maxbackup={{ audit_log_maxbackups }}
+ - --audit-log-maxsize={{ audit_log_maxsize }}
+ - --audit-policy-file={{ audit_policy_file }}
+{% endif %}
- --advertise-address={{ ip | default(ansible_default_ipv4.address) }}
- --etcd-servers={{ etcd_access_addresses }}
{% if etcd_events_cluster_enabled %}
@@ -184,6 +191,14 @@ spec:
- mountPath: /etc/ssl/certs/ca-bundle.crt
name: rhel-ca-bundle
readOnly: true
+{% endif %}
+{% if kubernetes_audit %}
+ - mountPath: {{ audit_log_mountpath }}
+ name: {{ audit_log_name }}
+ Writable: true
+ - mountPath: {{ audit_policy_mountpath }}
+ name: {{ audit_policy_name }}
+ Writable: true
{% endif %}
volumes:
- hostPath:
@@ -205,3 +220,11 @@ spec:
path: /etc/ssl/certs/ca-bundle.crt
name: rhel-ca-bundle
{% endif %}
+{% if kubernetes_audit %}
+ - hostPath:
+ path: {{ audit_log_hostpath }}
+ name: {{ audit_log_name }}
+ - hostPath:
+ path: {{ audit_policy_hostpath }}
+ name: {{ audit_policy_name }}
+{% endif %}
diff --git a/tests/files/gce_centos-weave-kubeadm.yml b/tests/files/gce_centos-weave-kubeadm.yml
index a410be3f2..199fa437c 100644
--- a/tests/files/gce_centos-weave-kubeadm.yml
+++ b/tests/files/gce_centos-weave-kubeadm.yml
@@ -9,5 +9,6 @@ startup_script: ""
kube_network_plugin: weave
kubeadm_enabled: true
deploy_netchecker: true
+kubernetes_audit: true
kubedns_min_replicas: 1
cloud_provider: gce
diff --git a/tests/files/gce_centos7-flannel-addons.yml b/tests/files/gce_centos7-flannel-addons.yml
index 1a03b0f9b..3dffa338f 100644
--- a/tests/files/gce_centos7-flannel-addons.yml
+++ b/tests/files/gce_centos7-flannel-addons.yml
@@ -8,6 +8,7 @@ mode: ha
kube_network_plugin: flannel
helm_enabled: true
efk_enabled: true
+kubernetes_audit: true
etcd_events_cluster_setup: true
local_volume_provisioner_enabled: true
etcd_deployment_type: host
--
GitLab