Initial commit

README.md

-# kubernetes-ansible
-Setup a kubernetes cluster
+This playbook deploys a whole kubernetes cluster, configures network overlay and some addons.
+
+# Download necessary binaries
+Note: a variable 'local_release_dir' defines where the binaries will be downloaded.
+Ensure you've enough disk space
+
+# Kubernetes
+Kubernetes services are configured with the nodePort type.
+eg: each node opoens the same tcp port and forwards the traffic to the target pod wherever it is located.
+
+master :
+  - apiserver :
+  Currently the apiserver listen on both secure and unsecure ports
+  todo, secure everything. Calico especially
+  - scheduler :
+  - controller :
+  - proxy
+node :
+  - kubelet :
+  kubelet is configured to call calico whenever a pod is created/destroyed
+  - proxy
+  configures all the forwarding rules
+
+# Overlay network
+You can choose between 2 network overlays. Only one must be chosen.
+flannel: gre/vxlan (layer 2) networking
+calico: bgp (layer 3) networking.
+
+# Loadbalancer
+The machine where ansible is ran must be allowed to access to the master ip on port 8080 (kubernetes api).
+Indeed it gathered the services definition in order to know which NodePort is configured.

cluster.yml

+---
+- hosts: downloader
+  sudo: no
+  roles:
+    - { role: download, tags: download }
+
+- hosts: k8s-cluster
+  roles:
+    - { role: etcd, tags: etcd }
+    - { role: docker, tags: docker }
+    - { role: overlay_network, tags: ['calico', 'flannel', 'network'] }
+    - { role: dnsmasq, tags: dnsmasq }
+
+- hosts: kube-master
+  roles:
+    - { role: kubernetes/master, tags: master }
+    - { role: addons, tags: addons }
+
+- hosts: kube-node
+  roles:
+    - { role: kubernetes/node, tags: node }

environments/dev/group_vars/all.yml

+# Directory where the binaries will be installed
+bin_dir: /usr/local/bin
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"

environments/dev/group_vars/k8s-cluster.yml

+# Users to create for basic auth in Kubernetes API via HTTP
+kube_users:
+  kube:
+    pass: changeme
+    role: admin
+  root:
+    pass: changeme
+    role: admin
+
+# Kubernetes cluster name, also will be used as DNS domain
+cluster_name: cluster.local
+   #
+# set this variable to calico if needed. keep it empty if flannel is used
+overlay_network_plugin: calico
+
+# Kubernetes internal network for services, unused block of space.
+kube_service_addresses: 10.233.0.0/18
+
+# internal network. When used, it will assign IP
+# addresses from this range to individual pods.
+# This network must be unused in your network infrastructure!
+overlay_network_subnet: 10.233.64.0/18
+
+# internal network total size (optional). This is the prefix of the
+# entire overlay network.  So the entirety of 4.0.0.0/16 must be
+# unused in your environment.
+# overlay_network_prefix: 18
+
+# internal network node size allocation (optional). This is the size allocated
+# to each node on your network.  With these defaults you should have
+# room for 4096 nodes with 254 pods per node.
+overlay_network_host_prefix: 24
+
+# Internal DNS configuration.
+# Kubernetes can create and mainatain its own DNS server to resolve service names
+# into appropriate IP addresses. It's highly advisable to run such DNS server,
+# as it greatly simplifies configuration of your applications - you can use
+# service names instead of magic environment variables.
+# You still must manually configure all your containers to use this DNS server,
+# Kubernetes won't do this for you (yet).
+
+# Upstream dns servers used by dnsmasq
+upstream_dns_servers:
+  - 8.8.8.8
+  - 4.4.8.8
+
+# Turn this varable to 'false' to disable whole DNS configuration.
+dns_setup: true
+dns_domain: "{{ cluster_name }}"
+
+# Ip address of the kubernetes dns service
+kube_dns_server: 10.233.0.10
+
+# Number of replicas of DNS instances started on kubernetes
+dns_replicas: 2
+
+# Set to 'false' to disable default Kubernetes UI setup
+enable_ui: true
+
+# Set to 'false' to disable default Elasticsearch + Kibana logging setup
+enable_logging: false
+
+# Set to "false' to disable default Monitoring (cAdvisor + heapster + influxdb + grafana)
+enable_monitoring: false
+ 
+# Set to 'false' to disable the docker garbage collection.
+# Every hour it removes images that are not used by containers and exited containers.
+enable_docker_gc: true

environments/dev/inventory

+[downloader]
+192.168.0.1
+
+[kube-master]
+# NB : the br_addr must be in the {{ calico_pool }} subnet
+# it will assign a /24 subnet per node
+192.168.0.1 br_addr=10.233.64.1
+
+[kube-node]
+192.168.0.2 br_addr=10.233.65.1
+192.168.0.3 br_addr=10.233.66.1
+192.168.0.4 br_addr=10.233.67.1
+
+[etcd]
+192.168.0.1
+
+[k8s-cluster:children]
+kube-node
+kube-master

environments/production/group_vars/all.yml

+# Directory where the binaries will be installed
+# bin_dir: /usr/local/bin
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+# local_release_dir: "/tmp/releases"

environments/production/group_vars/k8s-cluster.yml

+# Users to create for basic auth in Kubernetes API via HTTP
+# kube_users:
+#   kube:
+#     pass: changeme
+#     role: admin
+#   root:
+#     pass: changeme
+#     role: admin
+
+# Kubernetes cluster name, also will be used as DNS domain
+# cluster_name: cluster.local
+   #
+# set this variable to calico if needed. keep it empty if flannel is used
+# overlay_network_plugin: calico
+
+# Kubernetes internal network for services, unused block of space.
+# kube_service_addresses: 10.233.0.0/18
+
+# internal network. When used, it will assign IP
+# addresses from this range to individual pods.
+# This network must be unused in your network infrastructure!
+# overlay_network_subnet: 10.233.64.0/18
+
+# internal network total size (optional). This is the prefix of the
+# entire overlay network.  So the entirety of 4.0.0.0/16 must be
+# unused in your environment.
+# overlay_network_prefix: 18
+
+# internal network node size allocation (optional). This is the size allocated
+# to each node on your network.  With these defaults you should have
+# room for 4096 nodes with 254 pods per node.
+# overlay_network_host_prefix: 24
+
+# Internal DNS configuration.
+# Kubernetes can create and mainatain its own DNS server to resolve service names
+# into appropriate IP addresses. It's highly advisable to run such DNS server,
+# as it greatly simplifies configuration of your applications - you can use
+# service names instead of magic environment variables.
+# You still must manually configure all your containers to use this DNS server,
+# Kubernetes won't do this for you (yet).
+
+# Upstream dns servers used by dnsmasq
+# upstream_dns_servers:
+#   - 8.8.8.8
+#   - 4.4.8.8
+
+# Turn this varable to 'false' to disable whole DNS configuration.
+# dns_setup: true
+# dns_domain: "{{ cluster_name }}"
+
+# Ip address of the kubernetes dns service
+# kube_dns_server: 10.233.0.10
+
+# Number of replicas of DNS instances started on kubernetes
+# dns_replicas: 2
+
+# Set to 'false' to disable default Kubernetes UI setup
+# enable_ui: true
+
+# Set to 'false' to disable default Elasticsearch + Kibana logging setup
+# enable_logging: false
+
+# Set to "false' to disable default Monitoring (cAdvisor + heapster + influxdb + grafana)
+# enable_monitoring: false
+
+# Set to 'false' to disable the docker garbage collection.
+# Every hour it removes images that are not used by containers and exited containers.
+# enable_docker_gc: true

library/kube.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+DOCUMENTATION = """
+---
+module: kube
+short_description: Manage Kubernetes Cluster
+description:
+  - Create, replace, remove, and stop resources within a Kubernetes Cluster
+version_added: "2.0"
+options:
+  name:
+    required: false
+    default: null
+    description:
+      - The name associated with resource
+  filename:
+    required: false
+    default: null
+    description:
+      - The path and filename of the resource(s) definition file.
+  namespace:
+    required: false
+    default: null
+    description:
+      - The namespace associated with the resource(s)
+  resource:
+    required: false
+    default: null
+    description:
+      - The resource to perform an action on. pods (po), replicationControllers (rc), services (svc)
+  label:
+    required: false
+    default: null
+    description:
+      - The labels used to filter specific resources.
+  server:
+    required: false
+    default: null
+    description:
+      - The url for the API server that commands are executed against.
+  api_version:
+    required: false
+    choices: ['v1', 'v1beta3']
+    default: v1
+    description:
+      - The API version associated with cluster.
+  force:
+    required: false
+    default: false
+    description:
+      - A flag to indicate to force delete, replace, or stop.
+  all:
+    required: false
+    default: false
+    description:
+      - A flag to indicate delete all, stop all, or all namespaces when checking exists.
+  log_level:
+    required: false
+    default: 0
+    description:
+      - Indicates the level of verbosity of logging by kubectl.
+  state:
+    required: false
+    choices: ['present', 'absent', 'latest', 'reloaded', 'stopped']
+    default: present
+    description:
+      - present handles checking existence or creating if definition file provided,
+        absent handles deleting resource(s) based on other options,
+        latest handles creating ore updating based on existence,
+        reloaded handles updating resource(s) definition using definition file,
+        stopped handles stopping resource(s) based on other options.
+requirements:
+  - kubectl
+author: "Kenny Jones (@kenjones-cisco)"
+"""
+
+EXAMPLES = """
+- name: test nginx is present
+  kube: name=nginx resource=rc state=present
+
+- name: test nginx is stopped
+  kube: name=nginx resource=rc state=stopped
+
+- name: test nginx is absent
+  kube: name=nginx resource=rc state=absent
+
+- name: test nginx is present
+  kube: filename=/tmp/nginx.yml
+"""
+
+
+class KubeManager(object):
+
+    def __init__(self, module):
+
+        self.module = module
+
+        self.base_cmd = [module.get_bin_path('kubectl', True)]
+        self.api_version = module.params.get('api_version')
+
+        if self.api_version:
+            self.base_cmd.append('--api-version=' + self.api_version)
+
+        if module.params.get('server'):
+            self.base_cmd.append('--server=' + module.params.get('server'))
+
+        if module.params.get('log_level'):
+            self.base_cmd.append('--v=' + str(module.params.get('log_level')))
+
+        if module.params.get('namespace'):
+            self.base_cmd.append('--namespace=' + module.params.get('namespace'))
+
+        self.all = module.params.get('all')
+        self.force = module.params.get('force')
+        self.name = module.params.get('name')
+        self.filename = module.params.get('filename')
+        self.resource = module.params.get('resource')
+        self.label = module.params.get('label')
+
+    def _execute(self, cmd):
+        args = self.base_cmd + cmd
+        try:
+            rc, out, err = self.module.run_command(args)
+            if rc != 0:
+                self.module.fail_json(
+                    msg='error running kubectl (%s) command (rc=%d): %s' % (' '.join(args), rc, out or err))
+        except Exception as exc:
+            self.module.fail_json(
+                msg='error running kubectl (%s) command: %s' % (' '.join(args), str(exc)))
+        return out.splitlines()
+
+    def _execute_nofail(self, cmd):
+        args = self.base_cmd + cmd
+        rc, out, err = self.module.run_command(args)
+        if rc != 0:
+            return None
+        return out.splitlines()
+
+    def create(self, check=True):
+        if check and self.exists():
+            return []
+
+        cmd = ['create']
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to create')
+
+        cmd.append('--filename=' + self.filename)
+
+        return self._execute(cmd)
+
+    def replace(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['replace']
+        if self.api_version != 'v1':
+            cmd = ['update']
+
+        if self.force:
+            cmd.append('--force')
+
+        if not self.filename:
+            self.module.fail_json(msg='filename required to reload')
+
+        cmd.append('--filename=' + self.filename)
+
+        return self._execute(cmd)
+
+    def delete(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['delete']
+
+        if self.filename:
+            cmd.append('--filename=' + self.filename)
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to delete without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+        return self._execute(cmd)
+
+    def exists(self):
+        cmd = ['get']
+
+        if not self.resource:
+            return False
+
+        cmd.append(self.resource)
+
+        if self.name:
+            cmd.append(self.name)
+
+        cmd.append('--no-headers')
+
+        if self.label:
+            cmd.append('--selector=' + self.label)
+
+        if self.all:
+            cmd.append('--all-namespaces')
+
+        result = self._execute_nofail(cmd)
+        if not result:
+            return False
+        return True
+
+    def stop(self):
+
+        if not self.force and not self.exists():
+            return []
+
+        cmd = ['stop']
+
+        if self.filename:
+            cmd.append('--filename=' + self.filename)
+        else:
+            if not self.resource:
+                self.module.fail_json(msg='resource required to stop without filename')
+
+            cmd.append(self.resource)
+
+            if self.name:
+                cmd.append(self.name)
+
+            if self.label:
+                cmd.append('--selector=' + self.label)
+
+            if self.all:
+                cmd.append('--all')
+
+            if self.force:
+                cmd.append('--ignore-not-found')
+
+        return self._execute(cmd)
+
+
+def main():
+
+    module = AnsibleModule(
+        argument_spec=dict(
+            name=dict(),
+            filename=dict(),
+            namespace=dict(),
+            resource=dict(),
+            label=dict(),
+            server=dict(),
+            api_version=dict(default='v1', choices=['v1', 'v1beta3']),
+            force=dict(default=False, type='bool'),
+            all=dict(default=False, type='bool'),
+            log_level=dict(default=0, type='int'),
+            state=dict(default='present', choices=['present', 'absent', 'latest', 'reloaded', 'stopped']),
+            )
+        )
+
+    changed = False
+
+    manager = KubeManager(module)
+    state = module.params.get('state')
+
+    if state == 'present':
+        result = manager.create()
+
+    elif state == 'absent':
+        result = manager.delete()
+
+    elif state == 'reloaded':
+        result = manager.replace()
+
+    elif state == 'stopped':
+        result = manager.stop()
+
+    elif state == 'latest':
+        if manager.exists():
+            manager.force = True
+            result = manager.replace()
+        else:
+            result = manager.create(check=False)
+
+    else:
+        module.fail_json(msg='Unrecognized state %s.' % state)
+
+    if result:
+        changed = True
+    module.exit_json(changed=changed,
+                     msg='success: %s' % (' '.join(result))
+                     )
+
+
+from ansible.module_utils.basic import *  # noqa
+if __name__ == '__main__':
+    main()

roles/addons/defaults/main.yml

+---
+# defaults file for addons

roles/addons/files/es-rc.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: elasticsearch-logging-v1
+  namespace: kube-system
+  labels:
+    k8s-app: elasticsearch-logging
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 2
+  selector:
+    k8s-app: elasticsearch-logging
+    version: v1
+  template:
+    metadata:
+      labels:
+        k8s-app: elasticsearch-logging
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - image: gcr.io/google_containers/elasticsearch:1.7
+        name: elasticsearch-logging         
+        resources:
+          limits:
+            cpu: 100m
+        ports:
+        - containerPort: 9200
+          name: db
+          protocol: TCP
+        - containerPort: 9300
+          name: transport
+          protocol: TCP
+        volumeMounts:
+        - name: es-persistent-storage
+          mountPath: /data
+      volumes:
+      - name: es-persistent-storage
+        emptyDir: {}

roles/addons/files/es-svc.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch-logging
+  namespace: kube-system
+  labels:
+    k8s-app: elasticsearch-logging
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Elasticsearch"
+spec:
+  ports:
+  - port: 9200
+    protocol: TCP
+    targetPort: db
+  selector:
+    k8s-app: elasticsearch-logging

roles/addons/files/grafana-service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: monitoring-grafana
+  namespace: kube-system
+  labels: 
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Grafana"
+spec: 
+  type: NodePort
+  ports: 
+    - port: 80
+      targetPort: 8080
+  selector: 
+    k8s-app: influxGrafana
+

roles/addons/files/heapster-controller.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: monitoring-heapster-v8
+  namespace: kube-system
+  labels:
+    k8s-app: heapster
+    version: v8
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: heapster
+    version: v8
+  template:
+    metadata:
+      labels:
+        k8s-app: heapster
+        version: v8
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+        - image: gcr.io/google_containers/heapster:v0.17.0
+          name: heapster
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+          command:
+            - /heapster
+            - --source=kubernetes:''
+            - --sink=influxdb:http://monitoring-influxdb:8086

roles/addons/files/heapster-service.yaml

+kind: Service
+apiVersion: v1
+metadata:
+  name: monitoring-heapster
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Heapster"
+spec: 
+  type: NodePort
+  ports: 
+    - port: 80
+      targetPort: 8082
+  selector: 
+    k8s-app: heapster

roles/addons/files/influxdb-grafana-controller.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: monitoring-influx-grafana-v1
+  namespace: kube-system
+  labels: 
+    k8s-app: influxGrafana
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec: 
+  replicas: 1
+  selector: 
+    k8s-app: influxGrafana
+    version: v1
+  template: 
+    metadata: 
+      labels: 
+        k8s-app: influxGrafana
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec: 
+      containers: 
+        - image: gcr.io/google_containers/heapster_influxdb:v0.3
+          name: influxdb
+          resources:
+            limits:
+              cpu: 100m
+              memory: 200Mi
+          ports: 
+            - containerPort: 8083
+              hostPort: 8083
+            - containerPort: 8086
+              hostPort: 8086
+          volumeMounts:
+          - name: influxdb-persistent-storage
+            mountPath: /data
+        - image: gcr.io/google_containers/heapster_grafana:v0.7
+          name: grafana
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+          env: 
+            - name: INFLUXDB_EXTERNAL_URL
+              value: /api/v1/proxy/namespaces/kube-system/services/monitoring-influxdb:api/db/
+            - name: INFLUXDB_HOST
+              value: monitoring-influxdb
+            - name: INFLUXDB_PORT
+              value: "8086"
+      volumes:
+      - name: influxdb-persistent-storage
+        emptyDir: {}
+

roles/addons/files/influxdb-service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: monitoring-influxdb
+  namespace: kube-system
+  labels: 
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "InfluxDB"
+spec: 
+  ports: 
+    - name: http
+      port: 8083
+      targetPort: 8083
+    - name: api
+      port: 8086
+      targetPort: 8086
+  selector: 
+    k8s-app: influxGrafana
+

roles/addons/files/kibana-rc.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kibana-logging-v1
+  namespace: kube-system
+  labels:
+    k8s-app: kibana-logging
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: kibana-logging
+    version: v1
+  template:
+    metadata:
+      labels:
+        k8s-app: kibana-logging
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: kibana-logging
+        image: gcr.io/google_containers/kibana:1.3
+        resources:
+          limits:
+            cpu: 100m
+        env:
+          - name: "ELASTICSEARCH_URL"
+            value: "http://elasticsearch-logging:9200"
+        ports:
+        - containerPort: 5601
+          name: ui
+          protocol: TCP

roles/addons/files/kibana-svc.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana-logging
+  namespace: kube-system
+  labels:
+    k8s-app: kibana-logging
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "Kibana"
+spec:
+  type: NodePort
+  ports:
+  - port: 5601
+    protocol: TCP
+    targetPort: ui
+  selector:
+    k8s-app: kibana-logging

roles/addons/files/kube-system.yaml

+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kube-system

roles/addons/files/kube-ui-rc.yaml

+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: kube-ui-v1
+  namespace: kube-system
+  labels:
+    k8s-app: kube-ui
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    k8s-app: kube-ui
+    version: v1
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-ui
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: kube-ui
+        image: gcr.io/google_containers/kube-ui:v1.1
+        resources:
+          limits:
+            cpu: 100m
+            memory: 50Mi
+        ports:
+        - containerPort: 8080
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080
+          initialDelaySeconds: 30
+          timeoutSeconds: 5