Adding EFK logging stack

cluster.yml

@@ -63,3 +63,4 @@
   any_errors_fatal: true
   roles:
     - { role: kubernetes-apps, tags: apps }
+    - { role: kubernetes-apps/efk, tags: [ apps, efk ] }

inventory/group_vars/all.yml

@@ -204,3 +204,5 @@ kpm_packages: []
 rkt_version: 1.21.0
 etcd_deployment_type: docker
 kubelet_deployment_type: docker
+
+efk_enabled: false

roles/download/defaults/main.yml

@@ -82,6 +82,15 @@ kubedns_image_repo: "gcr.io/google_containers/kubedns-amd64"
 kubedns_image_tag: "{{ kubedns_version }}"
 test_image_repo: busybox
 test_image_tag: latest
+elasticsearch_version: "v2.4.1"
+elasticsearch_image_repo: "gcr.io/google_containers/elasticsearch"
+elasticsearch_image_tag: "{{ elasticsearch_version }}"
+fluentd_version: "1.22"
+fluentd_image_repo: "gcr.io/google_containers/fluentd-elasticsearch"
+fluentd_image_tag: "{{ fluentd_version }}"
+kibana_version: "v4.6.1"
+kibana_image_repo: "gcr.io/google_containers/kibana"
+kibana_image_tag: "{{ kibana_version }}"
 
 downloads:
   netcheck_server:
@@ -203,6 +212,21 @@ downloads:
     repo: "{{ exechealthz_image_repo }}"
     tag: "{{ exechealthz_image_tag }}"
     sha256: "{{ exechealthz_digest_checksum|default(None) }}"
+  elasticsearch:
+    container: true
+    repo: "{{ elasticsearch_image_repo }}"
+    tag: "{{ elasticsearch_image_tag }}"
+    sha256: "{{ elasticsearch_digest_checksum|default(None) }}"
+  fluentd:
+    container: true
+    repo: "{{ fluentd_image_repo }}"
+    tag: "{{ fluentd_image_tag }}"
+    sha256: "{{ fluentd_digest_checksum|default(None) }}"
+  kibana:
+    container: true
+    repo: "{{ kibana_image_repo }}"
+    tag: "{{ kibana_image_tag }}"
+    sha256: "{{ kibana_digest_checksum|default(None) }}"
 
 download:
   container: "{{ file.container|default('false') }}"

roles/kubernetes-apps/efk/elasticsearch/defaults/main.yml

+---
+elasticsearch_cpu_limit: 1000m 
+elasticsearch_mem_limit: 0M
+elasticsearch_cpu_requests: 100m
+elasticsearch_mem_requests: 0M
+elasticsearch_service_port: 9200

roles/kubernetes-apps/efk/elasticsearch/meta/main.yml

+dependencies:
+  - role: download
+    file: "{{ downloads.elasticsearch }}"
+# TODO: bradbeam add in curator
+#       https://github.com/Skillshare/kubernetes-efk/blob/master/configs/elasticsearch.yml#L94
+#  - role: download
+#    file: "{{ downloads.curator }}"

roles/kubernetes-apps/efk/elasticsearch/tasks/main.yml

+---
+- name: "ElasticSearch | Write ES deployment"
+  template: 
+    src: elasticsearch-deployment.yml.j2
+    dest: "{{ kube_config_dir }}/elasticsearch-deployment.yaml"
+  register: es_deployment_manifest
+
+- name: "ElasticSearch | Create ES deployment"
+  kube:
+    filename: "{{kube_config_dir}}/elasticsearch-deployment.yaml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "elasticsearch-logging-v1"
+    namespace: "{{system_namespace}}"
+    resource: "rc"
+    state: "{{ item | ternary('latest','present') }}"
+    #state: "latest"
+  with_items: "{{ es_deployment_manifest.changed }}"
+  run_once: true
+
+- name: "ElasticSearch | Write ES service "
+  template: 
+    src: elasticsearch-service.yml.j2
+    dest: "{{ kube_config_dir }}/elasticsearch-service.yaml"
+  register: es_service_manifest
+
+- name: "ElasticSearch | Create ES service"
+  kube:
+    filename: "{{kube_config_dir}}/elasticsearch-service.yaml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "elasticsearch-logging"
+    namespace: "{{system_namespace}}"
+    resource: "svc"
+    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ es_service_manifest.changed }}"
+  run_once: true

roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-deployment.yml.j2

+---
+# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: elasticsearch-logging-v1
+  namespace: "{{ system_namespace }}"
+  labels:
+    k8s-app: elasticsearch-logging
+    version: "{{ elasticsearch_image_tag }}"
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 2
+  selector:
+    k8s-app: elasticsearch-logging
+    version: "{{ elasticsearch_image_tag }}"
+  template:
+    metadata:
+      labels:
+        k8s-app: elasticsearch-logging
+        version: "{{ elasticsearch_image_tag }}"
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - image: "{{ elasticsearch_image_repo }}:{{ elasticsearch_image_tag }}"
+        name: elasticsearch-logging
+        resources:
+          # need more cpu upon initialization, therefore burstable class
+          limits:
+            cpu: {{ elasticsearch_cpu_limit }}
+{% if elasticsearch_mem_limit is defined and elasticsearch_mem_limit != "0M" %}
+            mem: {{ elasticsearch_mem_limit }}
+{% endif %}
+          requests:
+            cpu: {{ elasticsearch_cpu_requests }}
+{% if elasticsearch_mem_requests is defined and elasticsearch_mem_requests != "0M" %}
+            mem: {{ elasticsearch_mem_requests }}
+{% endif %}
+        ports:
+        - containerPort: 9200
+          name: db
+          protocol: TCP
+        - containerPort: 9300
+          name: transport
+          protocol: TCP
+        volumeMounts:
+        - name: es-persistent-storage
+          mountPath: /data
+      volumes:
+      - name: es-persistent-storage
+        emptyDir: {}

roles/kubernetes-apps/efk/elasticsearch/templates/elasticsearch-service.yml.j2

+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch-logging
+  namespace: "{{ system_namespace }}"
+  labels:
+    k8s-app: elasticsearch-logging
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Elasticsearch"
+spec:
+  ports:
+  - port: {{ elasticsearch_service_port }}
+    protocol: TCP
+    targetPort: db
+  selector:
+    k8s-app: elasticsearch-logging
+

roles/kubernetes-apps/efk/fluentd/defaults/main.yml

+---
+fluentd_cpu_limit: 0m 
+fluentd_mem_limit: 200Mi
+fluentd_cpu_requests: 100m
+fluentd_mem_requests: 200Mi

roles/kubernetes-apps/efk/fluentd/meta/main.yml

+dependencies:
+  - role: download
+    file: "{{ downloads.fluentd }}"

roles/kubernetes-apps/efk/fluentd/tasks/main.yml

+---
+- name: "Fluentd | Write fluentd daemonset"
+  template: 
+    src: fluentd-ds.yml.j2
+    dest: "{{ kube_config_dir }}/fluentd-ds.yaml"
+  register: fluentd_ds_manifest
+
+- name: "Fluentd | Create fluentd daemonset"
+  kube:
+    filename: "{{kube_config_dir}}/fluentd-ds.yaml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "fluentd-es-v{{ fluentd_version }}"
+    namespace: "{{system_namespace}}"
+    resource: "ds"
+    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ fluentd_ds_manifest.changed }}"
+  run_once: true

roles/kubernetes-apps/efk/fluentd/templates/fluentd-ds.yml.j2

+---
+# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-elasticsearch/es-controller.yaml
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: "fluentd-es-v{{ fluentd_version }}"
+  namespace: "{{ system_namespace }}"
+  labels:
+    k8s-app: fluentd-es
+    kubernetes.io/cluster-service: "true"
+    version: "v{{ fluentd_version }}"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-es
+        kubernetes.io/cluster-service: "true"
+        version: "v{{ fluentd_version }}"
+    spec:
+      containers:
+      - name: fluentd-es
+        image: "{{ fluentd_image_repo }}:{{ fluentd_image_tag }}"
+        command:
+          - '/bin/sh'
+          - '-c'
+          - '/usr/sbin/td-agent 2>&1 >> /var/log/fluentd.log'
+        resources:
+          limits:
+{% if fluentd_cpu_limit is defined and fluentd_cpu_limit != "0m" %}
+            mem: {{ fluentd_cpu_limit }}
+{% endif %}
+            memory: {{ fluentd_mem_limit }}
+          requests:
+            cpu:  {{ fluentd_cpu_requests }}
+            memory: {{ fluentd_mem_requests }}
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers

roles/kubernetes-apps/efk/kibana/defaults/main.yml

+---
+kibana_cpu_limit: 100m 
+kibana_mem_limit: 0M
+kibana_cpu_requests: 100m
+kibana_mem_requests: 0M
+kibana_service_port: 5601
+kibaba_base_url: "/api/v1/proxy/namespaces/kube-system/services/kibana-logging"

roles/kubernetes-apps/efk/kibana/meta/main.yml

+dependencies:
+  - role: download
+    file: "{{ downloads.kibana }}"

roles/kubernetes-apps/efk/kibana/tasks/main.yml

+---
+- name: "Kibana | Write Kibana deployment"
+  template: 
+    src: kibana-deployment.yml.j2
+    dest: "{{ kube_config_dir }}/kibana-deployment.yaml"
+  register: kibana_deployment_manifest
+
+- name: "Kibana | Create Kibana deployment"
+  kube:
+    filename: "{{kube_config_dir}}/kibana-deployment.yaml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "kibana-logging"
+    namespace: "{{system_namespace}}"
+    resource: "rc"
+    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ kibana_deployment_manifest.changed }}"
+  run_once: true
+
+- name: "Kibana | Write Kibana service "
+  template: 
+    src: kibana-service.yml.j2
+    dest: "{{ kube_config_dir }}/kibana-service.yaml"
+  register: kibana_service_manifest
+
+- name: "Kibana | Create Kibana service"
+  kube:
+    filename: "{{kube_config_dir}}/kibana-service.yaml"
+    kubectl: "{{bin_dir}}/kubectl"
+    name: "kibana-logging"
+    namespace: "{{system_namespace}}"
+    resource: "svc"
+    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ kibana_service_manifest.changed }}"
+  run_once: true

roles/kubernetes-apps/efk/kibana/templates/kibana-deployment.yml.j2

+---
+# https://raw.githubusercontent.com/kubernetes/kubernetes/v1.5.2/cluster/addons/fluentd-kibana/kibana-controller.yaml
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kibana-logging
+  namespace: "{{ system_namespace  }}"
+  labels:
+    k8s-app: kibana-logging
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kibana-logging
+  template:
+    metadata:
+      labels:
+        k8s-app: kibana-logging
+    spec:
+      containers:
+      - name: kibana-logging
+        image: "{{ kibana_image_repo }}:{{ kibana_image_tag }}"
+        resources:
+          # keep request = limit to keep this container in guaranteed class
+          limits:
+            cpu: {{ kibana_cpu_limit }}
+{% if kibana_mem_limit is defined and kibana_mem_limit != "0M" %}
+            mem: {{ kibana_mem_limit }}
+{% endif %}
+          requests:
+            cpu: {{ kibana_cpu_requests }}
+{% if kibana_mem_requests is defined and kibana_mem_requests != "0M" %}
+            mem: {{ kibana_mem_requests }}
+{% endif %}
+        env:
+          - name: "ELASTICSEARCH_URL"
+            value: "http://elasticsearch-logging:{{ elasticsearch_service_port }}"
+{% if kibana_base_url is defined and kibana_base_url != "" %}
+          - name: "KIBANA_BASE_URL"
+            value: "{{ kibana_base_url }}"
+{% endif %}
+        ports:
+        - containerPort: 5601
+          name: ui
+          protocol: TCP

roles/kubernetes-apps/efk/kibana/templates/kibana-service.yml.j2

+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana-logging
+  namespace: "{{ system_namespace }}"
+  labels:
+    k8s-app: kibana-logging
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Kibana"
+spec:
+  ports:
+  - port: {{ kibana_service_port }}
+    protocol: TCP
+    targetPort: ui
+  selector:
+    k8s-app: kibana-logging
+

roles/kubernetes-apps/efk/meta/main.yml

+dependencies:
+  - role: kubernetes-apps/efk/elasticsearch
+  - role: kubernetes-apps/efk/fluentd
+  - role: kubernetes-apps/efk/kibana

roles/kubernetes-apps/meta/main.yaml

@@ -18,3 +18,6 @@ dependencies:
     tags: [download, netchecker]
   - {role: kubernetes-apps/ansible, tags: apps}
   - {role: kubernetes-apps/kpm, tags: [apps, kpm]}
+  - role: kubernetes-apps/efk
+    when: efk_enabled
+    tags: [ apps, efk ]