Merge pull request #22 from ansibl8s/ha_master

- HA (kubernetes and etcd)
- Dockerize kubenertes components (api-server/scheduler/replica-manager/proxy)

roles/kubernetes/master/templates/proxy.kubeconfig.j2

-apiVersion: v1
-kind: Config
-current-context: proxy-to-{{ cluster_name }}
-preferences: {}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: proxy
-  name: proxy-to-{{ cluster_name }}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: http://{{ groups['kube-master'][0] }}:{{kube_master_insecure_port}}
-  name: {{ cluster_name }}
-users:
-- name: proxy
-  user:
-    token: {{ proxy_token }}

roles/kubernetes/master/templates/scheduler.j2

-###
-# kubernetes scheduler config
-
-# default config should be adequate
-
-# Add your own!
-KUBE_SCHEDULER_ARGS="--kubeconfig={{ kube_config_dir }}/scheduler.kubeconfig"

roles/kubernetes/master/templates/scheduler.kubeconfig.j2

-apiVersion: v1
-kind: Config
-current-context: scheduler-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: scheduler
-  name: scheduler-to-{{ cluster_name }}
-users:
-- name: scheduler
-  user:
-    token: {{ scheduler_token }}

roles/kubernetes/master/templates/systemd-init/kube-apiserver.service.j2

-[Unit]
-Description=Kubernetes API Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd2.service
-After=etcd2.service
-
-[Service]
-EnvironmentFile=/etc/network-environment
-EnvironmentFile=-/etc/kubernetes/config
-EnvironmentFile=-/etc/kubernetes/apiserver
-User=kube
-ExecStart={{ bin_dir }}/kube-apiserver \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_ETCD_SERVERS \
-	    $KUBE_API_ADDRESS \
-	    $KUBE_API_PORT \
-	    $KUBELET_PORT \
-	    $KUBE_ALLOW_PRIV \
-	    $KUBE_SERVICE_ADDRESSES \
-	    $KUBE_ADMISSION_CONTROL \
-	    $KUBE_RUNTIME_CONFIG \
-	    $KUBE_API_ARGS
-Restart=on-failure
-Type=notify
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-controller-manager.service.j2

-[Unit]
-Description=Kubernetes Controller Manager
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd2.service
-After=etcd2.service
-
-[Service]
-EnvironmentFile=-/etc/kubernetes/config
-EnvironmentFile=-/etc/kubernetes/controller-manager
-User=kube
-ExecStart={{ bin_dir }}/kube-controller-manager \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_CONTROLLER_MANAGER_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-proxy.service.j2

-[Unit]
-Description=Kubernetes Kube-Proxy Server
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-After=docker.service calico-node.service
-{% else %}
-After=docker.service
-{% endif %}
-
-[Service]
-EnvironmentFile=/etc/kubernetes/config
-EnvironmentFile=/etc/network-environment
-ExecStart={{ bin_dir }}/kube-proxy \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_PROXY_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/master/templates/systemd-init/kube-scheduler.service.j2

-[Unit]
-Description=Kubernetes Scheduler Plugin
-Documentation=https://github.com/GoogleCloudPlatform/kubernetes
-Requires=etcd2.service
-After=etcd2.service
-
-[Service]
-EnvironmentFile=-/etc/kubernetes/config
-EnvironmentFile=-/etc/kubernetes/scheduler
-User=kube
-ExecStart={{ bin_dir }}/kube-scheduler \
-	    $KUBE_LOGTOSTDERR \
-	    $KUBE_LOG_LEVEL \
-	    $KUBE_MASTER \
-	    $KUBE_SCHEDULER_ARGS
-Restart=on-failure
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

roles/kubernetes/common/defaults/main.yml → roles/kubernetes/node/defaults/main.yml

@@ -12,7 +12,7 @@ kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
 kube_config_dir: /etc/kubernetes
 
 # This is where all the cert scripts and certs will be located
-kube_cert_dir: "{{ kube_config_dir }}/certs"
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
 
 # This is where all of the bearer tokens will be stored
 kube_token_dir: "{{ kube_config_dir }}/tokens"
@@ -32,13 +32,18 @@ dns_domain: "{{ cluster_name }}"
 
 kube_proxy_mode: userspace
 
+# Temporary image, waiting for official google release
+#  hyperkube_image_repo: gcr.io/google_containers/hyperkube
+hyperkube_image_repo: quay.io/smana/hyperkube
+hyperkube_image_tag: v1.1.3
+
 # IP address of the DNS server.
 # Kubernetes will create a pod with several containers, serving as the DNS
 # server and expose it under this IP address. The IP address must be from
 # the range specified as kube_service_addresses. This magic will actually
 # pick the 10th ip address in the kube_service_addresses range and use that.
-# dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
+dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(253)|ipaddr('address') }}"
 
-# kube_api_runtime_config:
-#   - extensions/v1beta1/daemonsets=true
-#   - extensions/v1beta1/deployments=true
+kube_api_runtime_config:
+  - extensions/v1beta1/daemonsets=true
+  - extensions/v1beta1/deployments=true

roles/kubernetes/node/files/make-ssl.sh

+#!/bin/bash
+
+# Author: skahlouc@skahlouc-laptop
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-c <cloud_provider>] [-d <ssldir>] [-g <ssl_group>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -c | --cloud        : Cloud provider (GCE, AWS or AZURE)
+      -d | --ssldir       : Directory where the certificates will be installed
+      -g | --sslgrp       : Group of the certificates
+               
+               ex : 
+               $(basename $0) -f openssl.conf -c GCE -d /srv/ssl -g kube
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -c | --cloud) CLOUD=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;; 
+        -g | --group) SSLGRP="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/kubernetes/certs"
+fi
+if [ -z ${SSLGRP} ]; then
+    SSLGRP="kube-cert"
+fi
+
+#echo "config=$CONFIG, cloud=$CLOUD, certdir=$SSLDIR, certgroup=$SSLGRP"
+
+SUPPORTED_CLOUDS="GCE AWS AZURE"
+
+# TODO: Add support for discovery on other providers?
+if [ "${CLOUD}" == "GCE" ]; then
+  CLOUD_IP=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
+fi
+
+if [ "${CLOUD}" == "AWS" ]; then
+  CLOUD_IP=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
+fi
+
+if [ "${CLOUD}" == "AZURE" ]; then
+  CLOUD_IP=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
+fi
+
+tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR}"
+
+# Root CA
+openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
+openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=kube-ca" > /dev/null 2>&1
+
+# Apiserver
+openssl genrsa -out apiserver-key.pem 2048 > /dev/null 2>&1
+openssl req -new -key apiserver-key.pem -out apiserver.csr -subj "/CN=kube-apiserver" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in apiserver.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out apiserver.pem -days 365 -extensions v3_req -extfile ${CONFIG} > /dev/null 2>&1
+
+# Nodes and Admin
+for i in node admin; do
+    openssl genrsa -out ${i}-key.pem 2048 > /dev/null 2>&1
+    openssl req -new -key ${i}-key.pem -out ${i}.csr -subj "/CN=kube-${i}" > /dev/null 2>&1
+    openssl x509 -req -in ${i}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out ${i}.pem -days 365 > /dev/null 2>&1
+done
+
+# Install certs
+mv *.pem ${SSLDIR}/
+chgrp ${SSLGRP} ${SSLDIR}/*
+chmod 600 ${SSLDIR}/*-key.pem
+chown root:root ${SSLDIR}/*-key.pem

roles/kubernetes/node/handlers/main.yml

@@ -4,7 +4,6 @@
   notify:
     - reload systemd
     - restart reloaded-kubelet
-    - restart reloaded-proxy
 
 - name: reload systemd
   command: systemctl daemon-reload
@@ -19,14 +18,3 @@
   service:
     name: kubelet
     state: restarted
-
-- name: restart proxy
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-proxy
-
-- name: restart reloaded-proxy
-  service:
-    name: kube-proxy
-    state: restarted

roles/kubernetes/node/meta/main.yml

----
-dependencies:
-  - { role: kubernetes/common }

roles/kubernetes/node/tasks/config.yml

----
-- name: Get the node token values
-  slurp:
-    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
-  with_items:
-    - "system:kubelet"
-    - "system:proxy"
-  register: tokens
-  run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Set token facts
-  set_fact:
-    kubelet_token: "{{ tokens.results[0].content|b64decode }}"
-    proxy_token: "{{ tokens.results[1].content|b64decode }}"
-
-- name: Create kubelet environment vars dir
-  file: path=/etc/systemd/system/kubelet.service.d state=directory
-
-- name: Write kubelet config file
-  template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes
-  notify:
-    - restart kubelet
-
-- name: write the kubecfg (auth) file for kubelet
-  template: src=kubelet.kubeconfig.j2 dest={{ kube_config_dir }}/kubelet.kubeconfig backup=yes
-  notify:
-    - restart kubelet
-
-- name: Create proxy environment vars dir
-  file: path=/etc/systemd/system/kube-proxy.service.d state=directory
-
-- name: Write proxy config file
-  template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
-  notify:
-    - restart proxy
-
-- name: write the kubecfg (auth) file for kube-proxy
-  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
-  notify:
-    - restart proxy
-
-- name: Enable kubelet
-  service:
-    name: kubelet
-    enabled: yes
-    state: started
-
-- name: Enable proxy
-  service:
-    name: kube-proxy
-    enabled: yes
-    state: started

roles/kubernetes/node/tasks/gen_certs.yml

+---
+- name: certs | install cert generation script
+  copy:
+    src=make-ssl.sh
+    dest={{ kube_script_dir }}
+    mode=0500
+  changed_when: false
+
+- name: certs | write openssl config
+  template:
+    src: "openssl.conf.j2"
+    dest: "{{ kube_config_dir }}/.openssl.conf"
+
+- name: certs | run cert generation script
+  shell: >
+    {{ kube_script_dir }}/make-ssl.sh
+    -f {{ kube_config_dir }}/.openssl.conf
+    -g {{ kube_cert_group }}
+    -d {{ kube_cert_dir }}
+  args:
+    creates: "{{ kube_cert_dir }}/apiserver.pem"
+
+- name: certs | check certificate permissions
+  file:
+    path={{ kube_cert_dir }}
+    group={{ kube_cert_group }}
+    owner=kube
+    recurse=yes

roles/kubernetes/common/tasks/gen_tokens.yml → roles/kubernetes/node/tasks/gen_tokens.yml

@@ -10,21 +10,17 @@
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ "system:controller_manager", "system:scheduler", "system:kubectl", 'system:proxy' ]
-    - "{{ groups['kube-master'][0] }}"
+    - [ "system:kubectl" ]
+    - "{{ groups['kube-master'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
-  notify:
-    - restart daemons
 
 - name: tokens | generate tokens for node components
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
   environment:
     TOKEN_DIR: "{{ kube_token_dir }}"
   with_nested:
-    - [ 'system:kubelet', 'system:proxy' ]
+    - [ 'system:kubelet' ]
     - "{{ groups['kube-node'] }}"
   register: gentoken
   changed_when: "'Added' in gentoken.stdout"
-  notify:
-    - restart daemons

roles/kubernetes/node/tasks/install.yml

 ---
-- name: Write kube-proxy systemd init file
-  template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
-  notify: restart daemons
-
 - name: Write kubelet systemd init file
-  template: src=systemd-init/kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
-  notify: restart daemons
+  template: src=kubelet.service.j2 dest=/etc/systemd/system/kubelet.service backup=yes
+  notify: restart kubelet
 
-- name: Install kubernetes binaries
+- name: Install kubelet binary
   copy:
-     src={{ local_release_dir }}/kubernetes/bin/{{ item }}
+     src={{ local_release_dir }}/kubernetes/bin/kubelet
      dest={{ bin_dir }}
      owner=kube
      mode=u+x
-  with_items:
-    - kube-proxy
-    - kubelet
   notify:
-    - restart daemons
+    - restart kubelet

roles/kubernetes/node/tasks/main.yml

 ---
+- name: create kubernetes config directory
+  file: path={{ kube_config_dir }} state=directory
+
+- name: create kubernetes script directory
+  file: path={{ kube_script_dir }} state=directory
+
+- name: Make sure manifest directory exists
+  file: path={{ kube_manifest_dir }} state=directory
+
+- include: secrets.yml
+  tags:
+    - secrets
+
 - include: install.yml
-- include: config.yml
+
+- name: write the global config file
+  template:
+    src: config.j2
+    dest: "{{ kube_config_dir }}/config"
+  notify:
+    - restart kubelet
+
+- name: Create kubelet environment vars dir
+  file: path=/etc/systemd/system/kubelet.service.d state=directory
+
+- name: Write kubelet config file
+  template: src=kubelet.j2 dest=/etc/systemd/system/kubelet.service.d/10-kubelet.conf backup=yes
+  notify:
+    - restart kubelet
+
+- name: write the kubecfg (auth) file for kubelet
+  template: src=node-kubeconfig.yaml.j2 dest={{ kube_config_dir }}/node-kubeconfig.yaml backup=yes
+  notify:
+    - restart kubelet
+
+- name: Write proxy manifest
+  template: 
+    src: manifests/kube-proxy.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-proxy.manifest"
+
+- name: Enable kubelet
+  service:
+    name: kubelet
+    enabled: yes
+    state: started
+
 - include: temp_workaround.yml

roles/kubernetes/common/tasks/secrets.yml → roles/kubernetes/node/tasks/secrets.yml

@@ -29,26 +29,36 @@
   run_once: true
   when: inventory_hostname == groups['kube-master'][0]
 
-- name: Read back the CA certificate
-  slurp:
-    src: "{{ kube_cert_dir }}/ca.crt"
-  register: ca_cert
+- include: gen_tokens.yml
   run_once: true
-  delegate_to: "{{ groups['kube-master'][0] }}"
+  when: inventory_hostname == groups['kube-master'][0]
 
-- name: certs | register the CA certificate as a fact for later use
-  set_fact:
-    kube_ca_cert: "{{ ca_cert.content|b64decode }}"
+# Sync certs between nodes
+- user:
+    name: '{{ansible_user_id}}'
+    generate_ssh_key: yes
+  delegate_to: "{{ groups['kube-master'][0] }}"
+  run_once: yes
 
-- name: certs | write CA certificate everywhere
-  copy: content="{{ kube_ca_cert }}" dest="{{ kube_cert_dir }}/ca.crt"
-  notify:
-    - restart daemons
+- name: 'get ssh keypair'
+  slurp: path=~/.ssh/id_rsa.pub
+  register: public_key
+  delegate_to: "{{ groups['kube-master'][0] }}"
 
-- debug: msg="{{groups['kube-master'][0]}} == {{inventory_hostname}}"
-  tags:
-    - debug
+- name: 'setup keypair on nodes'
+  authorized_key:
+    user: '{{ansible_user_id}}'
+    key: "{{public_key.content|b64decode }}"
 
-- include: gen_tokens.yml
-  run_once: true
-  when: inventory_hostname == groups['kube-master'][0]
+- name: synchronize certificates for nodes
+  synchronize:
+    src: "{{ item }}"
+    dest: "{{ kube_cert_dir }}"
+    recursive: yes
+    delete: yes
+    rsync_opts: [ '--one-file-system']
+  with_items:
+    - "{{ kube_cert_dir}}/ca.pem"
+    - "{{ kube_cert_dir}}/node.pem"
+    - "{{ kube_cert_dir}}/node-key.pem"
+  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/temp_workaround.yml

-- name: Warning Temporary workaround !!! Disable kubelet and kube-proxy on node startup
-  service: name={{ item }} enabled=no
-  with_items:
-    - kubelet
-    - kube-proxy
+- name: Warning Temporary workaround !!! Disable kubelet on node startup
+  service: name=kubelet enabled=no

roles/kubernetes/common/templates/config.j2 → roles/kubernetes/node/templates/config.j2

@@ -17,10 +17,10 @@
 KUBE_LOGTOSTDERR="--logtostderr=true"
 
 # journal message level, 0 is debug
-KUBE_LOG_LEVEL="--v=5"
+KUBE_LOG_LEVEL="--v={{ kube_log_level | default('2') }}"
 
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow_privileged=true"
 
 # How the replication controller, scheduler, and proxy
-KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}"
+KUBE_MASTER="--master=https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}"