Merge pull request #22 from ansibl8s/ha_master

- HA (kubernetes and etcd)
- Dockerize kubenertes components (api-server/scheduler/replica-manager/proxy)

roles/etcd/templates/systemd-etcd2.service.j2

@@ -6,7 +6,11 @@ Conflicts=etcd.service
 User=etcd
 Environment=ETCD_DATA_DIR=/var/lib/etcd2
 Environment=ETCD_NAME=%m
+{% if inventory_hostname in groups['etcd'] %}
 ExecStart={{ bin_dir }}/etcd2
+{% else %}
+ExecStart={{ bin_dir }}/etcd2 -proxy on
+{% endif %}
 Restart=always
 RestartSec=10s
 LimitNOFILE=40000

roles/kubernetes/common/files/make-ca-cert.sh

-#!/bin/bash
-
-# Copyright 2014 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Caller should set in the ev:
-# MASTER_IP - this may be an ip or things like "_use_gce_external_ip_"
-# DNS_DOMAIN - which will be passed to minions in --cluster_domain
-# SERVICE_CLUSTER_IP_RANGE - where all service IPs are allocated
-# MASTER_NAME - I'm not sure what it is...
-
-# Also the following will be respected
-# CERT_DIR - where to place the finished certs
-# CERT_GROUP - who the group owner of the cert files should be
-
-cert_ip="${MASTER_IP:="${1}"}"
-master_name="${MASTER_NAME:="kubernetes"}"
-service_range="${SERVICE_CLUSTER_IP_RANGE:="10.0.0.0/16"}"
-dns_domain="${DNS_DOMAIN:="cluster.local"}"
-cert_dir="${CERT_DIR:-"/srv/kubernetes"}"
-cert_group="${CERT_GROUP:="kube-cert"}"
-
-# The following certificate pairs are created:
-#
-#  - ca (the cluster's certificate authority)
-#  - server
-#  - kubelet
-#  - kubecfg (for kubectl)
-#
-# TODO(roberthbailey): Replace easyrsa with a simple Go program to generate
-# the certs that we need.
-
-# TODO: Add support for discovery on other providers?
-if [ "$cert_ip" == "_use_gce_external_ip_" ]; then
-  cert_ip=$(curl -s -H Metadata-Flavor:Google http://metadata.google.internal./computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip)
-fi
-
-if [ "$cert_ip" == "_use_aws_external_ip_" ]; then
-  cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
-fi
-
-if [ "$cert_ip" == "_use_azure_dns_name_" ]; then
-  cert_ip=$(uname -n | awk -F. '{ print $2 }').cloudapp.net
-fi
-
-tmpdir=$(mktemp -d --tmpdir kubernetes_cacert.XXXXXX)
-trap 'rm -rf "${tmpdir}"' EXIT
-cd "${tmpdir}"
-
-# TODO: For now, this is a patched tool that makes subject-alt-name work, when
-# the fix is upstream  move back to the upstream easyrsa.  This is cached in GCS
-# but is originally taken from:
-#   https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
-#
-# To update, do the following:
-# curl -o easy-rsa.tar.gz https://github.com/brendandburns/easy-rsa/archive/master.tar.gz
-# gsutil cp easy-rsa.tar.gz gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
-# gsutil acl ch -R -g all:R gs://kubernetes-release/easy-rsa/easy-rsa.tar.gz
-#
-# Due to GCS caching of public objects, it may take time for this to be widely
-# distributed.
-
-# Calculate the first ip address in the service range
-octects=($(echo "${service_range}" | sed -e 's|/.*||' -e 's/\./ /g'))
-((octects[3]+=1))
-service_ip=$(echo "${octects[*]}" | sed 's/ /./g')
-
-# Determine appropriete subject alt names
-sans="IP:${cert_ip},IP:${service_ip},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.${dns_domain},DNS:${master_name}"
-
-curl -L -O https://storage.googleapis.com/kubernetes-release/easy-rsa/easy-rsa.tar.gz > /dev/null 2>&1
-tar xzf easy-rsa.tar.gz > /dev/null
-cd easy-rsa-master/easyrsa3
-
-(./easyrsa init-pki > /dev/null 2>&1
- ./easyrsa --batch "--req-cn=${cert_ip}@$(date +%s)" build-ca nopass > /dev/null 2>&1
- ./easyrsa --subject-alt-name="${sans}" build-server-full "${master_name}" nopass > /dev/null 2>&1
- ./easyrsa build-client-full kubelet nopass > /dev/null 2>&1
- ./easyrsa build-client-full kubecfg nopass > /dev/null 2>&1) || {
- # If there was an error in the subshell, just die.
- # TODO(roberthbailey): add better error handling here
- echo "=== Failed to generate certificates: Aborting ==="
- exit 2
- }
-
-mkdir -p "$cert_dir"
-
-cp -p pki/ca.crt "${cert_dir}/ca.crt"
-cp -p "pki/issued/${master_name}.crt" "${cert_dir}/server.crt" > /dev/null 2>&1
-cp -p "pki/private/${master_name}.key" "${cert_dir}/server.key" > /dev/null 2>&1
-cp -p pki/issued/kubecfg.crt "${cert_dir}/kubecfg.crt"
-cp -p pki/private/kubecfg.key "${cert_dir}/kubecfg.key"
-cp -p pki/issued/kubelet.crt "${cert_dir}/kubelet.crt"
-cp -p pki/private/kubelet.key "${cert_dir}/kubelet.key"
-
-CERTS=("ca.crt" "server.key" "server.crt" "kubelet.key" "kubelet.crt" "kubecfg.key" "kubecfg.crt")
-for cert in "${CERTS[@]}"; do
-  chgrp "${cert_group}" "${cert_dir}/${cert}"
-  chmod 660 "${cert_dir}/${cert}"
-done

roles/kubernetes/common/meta/main.yml

----
-dependencies:
-  - { role: etcd }

roles/kubernetes/common/tasks/gen_certs.yml

----
-#- name: Get create ca cert script from Kubernetes
-#  get_url:
-#    url=https://raw.githubusercontent.com/GoogleCloudPlatform/kubernetes/master/cluster/saltbase/salt/generate-cert/make-ca-cert.sh
-#    dest={{ kube_script_dir }}/make-ca-cert.sh mode=0500
-#    force=yes
-
-- name: certs | install cert generation script
-  copy:
-    src=make-ca-cert.sh
-    dest={{ kube_script_dir }}
-    mode=0500
-  changed_when: false
-
-# FIXME This only generates a cert for one master...
-- name: certs | run cert generation script
-  command:
-    "{{ kube_script_dir }}/make-ca-cert.sh {{ inventory_hostname }}"
-  args:
-    creates: "{{ kube_cert_dir }}/server.crt"
-  environment:
-    MASTER_IP: "{{ hostvars[inventory_hostname]['ip'] | default(hostvars[inventory_hostname]['ansible_default_ipv4']['address']) }}"
-    MASTER_NAME: "{{ inventory_hostname }}"
-    DNS_DOMAIN: "{{ dns_domain }}"
-    SERVICE_CLUSTER_IP_RANGE: "{{ kube_service_addresses }}"
-    CERT_DIR: "{{ kube_cert_dir }}"
-    CERT_GROUP: "{{ kube_cert_group }}"
-
-- name: certs | check certificate permissions
-  file:
-    path={{ item }}
-    group={{ kube_cert_group }}
-    owner=kube
-    mode=0440
-  with_items:
-    - "{{ kube_cert_dir }}/ca.crt"
-    - "{{ kube_cert_dir }}/server.crt"
-    - "{{ kube_cert_dir }}/server.key"
-    - "{{ kube_cert_dir }}/kubecfg.crt"
-    - "{{ kube_cert_dir }}/kubecfg.key"
-    - "{{ kube_cert_dir }}/kubelet.crt"
-    - "{{ kube_cert_dir }}/kubelet.key"

roles/kubernetes/common/tasks/main.yml

----
-- name: define alias command for kubectl all
-  lineinfile:
-    dest=/etc/bash.bashrc
-    line="alias kball='{{ bin_dir }}/kubectl --all-namespaces -o wide'"
-    regexp='^alias kball=.*$'
-    state=present
-    insertafter=EOF
-    create=True
-
-- name: create kubernetes config directory
-  file: path={{ kube_config_dir }} state=directory
-
-- name: create kubernetes script directory
-  file: path={{ kube_script_dir }} state=directory
-
-- name: Make sure manifest directory exists
-  file: path={{ kube_manifest_dir }} state=directory
-
-- name: write the global config file
-  template:
-    src: config.j2
-    dest: "{{ kube_config_dir }}/config"
-  notify:
-    - restart daemons
-
-- include: secrets.yml
-  tags:
-    - secrets

roles/kubernetes/master/handlers/main.yml

 ---
-- name: restart daemons
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-scheduler
-    - restart reloaded-controller-manager
-    - restart reloaded-apiserver
-    - restart reloaded-proxy
-
 - name: reload systemd
   command: systemctl daemon-reload
 
-- name: restart apiserver
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-apiserver
-
-- name: restart reloaded-apiserver
-  service:
-    name: kube-apiserver
-    state: restarted
-
-- name: restart controller-manager
-  command: /bin/true
-  notify:
-    - reload systemd
-    - restart reloaded-controller-manager
-
-- name: restart reloaded-controller-manager
-  service:
-    name: kube-controller-manager
-    state: restarted
-
-- name: restart scheduler
+- name: restart kubelet
   command: /bin/true
   notify:
     - reload systemd
-    - restart reloaded-scheduler
+    - restart reloaded-kubelet
 
-- name: restart reloaded-scheduler
+- name: restart reloaded-kubelet
   service:
-    name: kube-scheduler
+    name: kubelet
     state: restarted
 
 - name: restart proxy

roles/kubernetes/master/meta/main.yml

 ---
 dependencies:
-  - { role: kubernetes/common }
+  - { role: etcd }
+  - { role: kubernetes/node }

roles/kubernetes/master/tasks/config.yml

----
-- name: get the node token values from token files
-  slurp:
-    src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
-  with_items:
-    - "system:controller_manager"
-    - "system:scheduler"
-    - "system:kubectl"
-    - "system:proxy"
-  register: tokens
-  delegate_to: "{{ groups['kube-master'][0] }}"
-
-- name: Set token facts
-  set_fact:
-    controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
-    scheduler_token: "{{ tokens.results[1].content|b64decode }}"
-    kubectl_token: "{{ tokens.results[2].content|b64decode }}"
-    proxy_token: "{{ tokens.results[3].content|b64decode }}"
-
-- name: write the config files for api server
-  template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver backup=yes
-  notify:
-    - restart apiserver
-
-- name: write config file for controller-manager
-  template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager backup=yes
-  notify:
-    - restart controller-manager
-
-- name: write the kubecfg (auth) file for controller-manager
-  template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig backup=yes
-  notify:
-    - restart controller-manager
-
-- name: write the config file for scheduler
-  template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler backup=yes
-  notify:
-    - restart scheduler
-
-- name: write the kubecfg (auth) file for scheduler
-  template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig backup=yes
-  notify:
-    - restart scheduler
-
-- name: write the kubecfg (auth) file for kubectl
-  template: src=kubectl.kubeconfig.j2 dest={{ kube_config_dir }}/kubectl.kubeconfig backup=yes
-
-- name: Copy kubectl bash completion
-  copy: src=kubectl_bash_completion.sh dest=/etc/bash_completion.d/kubectl.sh
-
-- name: Create proxy environment vars dir
-  file: path=/etc/systemd/system/kube-proxy.service.d state=directory
-
-- name: Write proxy config file
-  template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
-  notify:
-    - restart proxy
-
-- name: write the kubecfg (auth) file for proxy
-  template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
-
-- name: populate users for basic auth in API
-  lineinfile:
-    dest: "{{ kube_users_dir }}/known_users.csv"
-    create: yes
-    line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
-    backup: yes
-  with_dict: "{{ kube_users }}"
-  notify:
-    - restart apiserver
-
-- name: Enable controller-manager
-  service:
-    name: kube-controller-manager
-    enabled: yes
-    state: started
-
-- name: Enable scheduler
-  service:
-    name: kube-scheduler
-    enabled: yes
-    state: started
-
-- name: Enable kube-proxy
-  service:
-    name: kube-proxy
-    enabled: yes
-    state: started
-
-- name: Enable apiserver
-  service:
-    name: kube-apiserver
-    enabled: yes
-    state: started

roles/kubernetes/master/tasks/install.yml

----
-- name: Write kube-apiserver systemd init file
-  template: src=systemd-init/kube-apiserver.service.j2 dest=/etc/systemd/system/kube-apiserver.service backup=yes
-  notify: restart apiserver
-
-- name: Write kube-controller-manager systemd init file
-  template: src=systemd-init/kube-controller-manager.service.j2 dest=/etc/systemd/system/kube-controller-manager.service backup=yes
-  notify: restart controller-manager
-
-- name: Write kube-scheduler systemd init file
-  template: src=systemd-init/kube-scheduler.service.j2 dest=/etc/systemd/system/kube-scheduler.service backup=yes
-  notify: restart scheduler
-
-- name: Write kube-proxy systemd init file
-  template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
-  notify: restart proxy
-
-- name: Install kubernetes binaries
-  copy:
-     src={{ local_release_dir }}/kubernetes/bin/{{ item }}
-     dest={{ bin_dir }}
-     owner=kube
-     mode=u+x
-  with_items:
-    - kube-apiserver
-    - kube-controller-manager
-    - kube-scheduler
-    - kube-proxy
-    - kubectl
-  notify:
-    - restart daemons
-
-- name: Allow apiserver to bind on both secure and insecure ports
-  shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver

roles/kubernetes/master/tasks/main.yml

 ---
-- include: install.yml
-- include: config.yml
+- name: Install kubectl binary
+  copy:
+     src={{ local_release_dir }}/kubernetes/bin/kubectl
+     dest={{ bin_dir }}
+     owner=kube
+     mode=u+x
+  notify:
+    - restart daemons
+
+- name: Copy kubectl bash completion
+  copy:
+    src: kubectl_bash_completion.sh
+    dest: /etc/bash_completion.d/kubectl.sh
+
+- name: populate users for basic auth in API
+  lineinfile:
+    dest: "{{ kube_users_dir }}/known_users.csv"
+    create: yes
+    line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
+    backup: yes
+  with_dict: "{{ kube_users }}"
+
+# Sync masters
+- name: synchronize auth directories for masters
+  synchronize:
+    src: "{{ item }}"
+    dest: "{{ kube_config_dir }}"
+    recursive: yes
+    delete: yes
+    rsync_opts: [ '--one-file-system']
+  with_items:
+    - "{{ kube_token_dir }}"
+    - "{{ kube_cert_dir }}"
+    - "{{ kube_users_dir }}"
+  delegate_to: "{{ groups['kube-master'][0] }}"
+
+# Write manifests
+- name: Write kube-apiserver manifest
+  template:
+    src: manifests/kube-apiserver.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-apisever.manifest"
+  notify:
+    - restart kubelet
+
+- meta: flush_handlers
+
+- name: wait for the apiserver to be running (pulling image and running container)
+  wait_for:
+    port: "{{kube_apiserver_insecure_port}}"
+    delay: 10
+
+- name: install required python module 'httplib2'
+  apt:
+    name: "python-httplib2"
+    state: present
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Create 'kube-system' namespace
+  uri:
+    url: http://{{ groups['kube-master'][0]}}:{{ kube_apiserver_insecure_port }}/api/v1/namespaces
+    method: POST
+    body: '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system"}}'
+    status_code: 201,409
+    body_format: json
+  run_once: yes
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Write kube-controller-manager manifest
+  template:
+    src: manifests/kube-controller-manager.manifest.j2
+    dest: "{{ kube_config_dir }}/kube-controller-manager.manifest"
+
+- name: Write kube-scheduler manifest
+  template:
+    src: manifests/kube-scheduler.manifest.j2
+    dest: "{{ kube_config_dir }}/kube-scheduler.manifest"
+
+- name: Write podmaster manifest
+  template:
+    src: manifests/kube-podmaster.manifest.j2
+    dest: "{{ kube_manifest_dir }}/kube-podmaster.manifest"

roles/kubernetes/master/templates/apiserver.j2

-###
-# kubernetes system config
-#
-# The following values are used to configure the kube-apiserver
-#
-
-# The address on the local server to listen to.
-KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
-
-# The port on the local server to listen on.
-KUBE_API_PORT="--insecure-port={{kube_master_insecure_port}} --secure-port={{ kube_master_port }}"
-
-# KUBELET_PORT="--kubelet_port=10250"
-
-# Address range to use for services
-KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
-
-# Location of the etcd cluster
-KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
-
-# default admission control policies
-KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
-
-# RUNTIME API CONFIGURATION (e.g. enable extensions)
-KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}"
-
-# Add you own!
-KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.crt --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/server.crt"

roles/kubernetes/master/templates/controller-manager.j2

-###
-# The following values are used to configure the kubernetes controller-manager
-
-# defaults from config and apiserver should be adequate
-
-KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig={{ kube_config_dir }}/controller-manager.kubeconfig --service_account_private_key_file={{ kube_cert_dir }}/server.key --root_ca_file={{ kube_cert_dir }}/ca.crt"

roles/kubernetes/master/templates/controller-manager.kubeconfig.j2

-apiVersion: v1
-kind: Config
-current-context: controller-manager-to-{{ cluster_name }}
-preferences: {}
-clusters:
-- cluster:
-    certificate-authority: {{ kube_cert_dir }}/ca.crt
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
-  name: {{ cluster_name }}
-contexts:
-- context:
-    cluster: {{ cluster_name }}
-    user: controller-manager
-  name: controller-manager-to-{{ cluster_name }}
-users:
-- name: controller-manager
-  user:
-    token: {{ controller_manager_token }}

roles/kubernetes/master/templates/kubectl.kubeconfig.j2 → roles/kubernetes/master/templates/kubectl-kubeconfig.yaml.j2

@@ -4,8 +4,8 @@ current-context: kubectl-to-{{ cluster_name }}
 preferences: {}
 clusters:
 - cluster:
-    certificate-authority-data: {{ kube_ca_cert|b64encode }}
-    server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
+    certificate-authority-data: {{ kube_node_cert|b64encode }}
+    server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}
   name: {{ cluster_name }}
 contexts:
 - context:

roles/kubernetes/master/templates/manifests/kube-apiserver.manifest.j2

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-apiserver
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-apiserver
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - apiserver
+    - --insecure-bind-address=0.0.0.0
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+    - --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
+    - --service-cluster-ip-range={{ kube_service_addresses }}
+    - --client-ca-file={{ kube_cert_dir }}/ca.pem
+    - --basic-auth-file={{ kube_users_dir }}/known_users.csv
+    - --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
+    - --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --secure-port={{ kube_apiserver_port }}
+    - --insecure-port={{ kube_apiserver_insecure_port }}
+{% if kube_api_runtime_config is defined %}
+{%   for conf in kube_api_runtime_config %}
+    - --runtime-config={{ conf }}
+{%   endfor %}
+{% endif %}
+    - --token-auth-file={{ kube_token_dir }}/known_tokens.csv
+    - --v={{ kube_log_level | default('2') }}
+    - --allow-privileged=true
+    ports:
+    - containerPort: {{ kube_apiserver_port }}
+      hostPort: {{ kube_apiserver_port }}
+      name: https
+    - containerPort: {{ kube_apiserver_insecure_port }}
+      hostPort: {{ kube_apiserver_insecure_port }}
+      name: local
+    volumeMounts:
+    - mountPath: {{ kube_config_dir }}
+      name: kubernetes-config
+      readOnly: true
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: {{ kube_config_dir }}
+    name: kubernetes-config
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: ssl-certs-host

roles/kubernetes/master/templates/manifests/kube-controller-manager.manifest.j2

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-controller-manager
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-controller-manager
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - controller-manager
+    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+    - --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
+    - --root-ca-file={{ kube_cert_dir }}/ca.pem
+    - --v={{ kube_log_level | default('2') }}
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 10252
+      initialDelaySeconds: 15
+      timeoutSeconds: 1
+    volumeMounts:
+    - mountPath: {{ kube_cert_dir }}
+      name: ssl-certs-kubernetes
+      readOnly: true
+    - mountPath: /etc/ssl/certs
+      name: ssl-certs-host
+      readOnly: true
+  volumes:
+  - hostPath:
+      path: {{ kube_cert_dir }}
+    name: ssl-certs-kubernetes
+  - hostPath:
+      path: /usr/share/ca-certificates
+    name: ssl-certs-host

roles/kubernetes/master/templates/manifests/kube-podmaster.manifest.j2

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-podmaster
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: scheduler-elector
+    image: gcr.io/google_containers/podmaster:1.1
+    command:
+    - /podmaster
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+    - --key=scheduler
+    - --source-file={{ kube_config_dir}}/kube-scheduler.manifest
+    - --dest-file={{ kube_manifest_dir }}/kube-scheduler.manifest
+    volumeMounts:
+    - mountPath: {{ kube_config_dir }}
+      name: manifest-src
+      readOnly: true
+    - mountPath: {{ kube_manifest_dir }}
+      name: manifest-dst
+  - name: controller-manager-elector
+    image: gcr.io/google_containers/podmaster:1.1
+    command:
+    - /podmaster
+    - --etcd-servers={% for srv in groups['etcd'] %}http://{{ srv }}:2379{% if not loop.last %},{% endif %}{% endfor %}
+
+    - --key=controller
+    - --source-file={{ kube_config_dir }}/kube-controller-manager.manifest
+    - --dest-file={{ kube_manifest_dir }}/kube-controller-manager.manifest
+    terminationMessagePath: /dev/termination-log
+    volumeMounts:
+    - mountPath: {{ kube_config_dir }}
+      name: manifest-src
+      readOnly: true
+    - mountPath: {{ kube_manifest_dir }}
+      name: manifest-dst
+  volumes:
+  - hostPath:
+      path: {{ kube_config_dir }}
+    name: manifest-src
+  - hostPath:
+      path: {{ kube_manifest_dir }}
+    name: manifest-dst

roles/kubernetes/master/templates/manifests/kube-scheduler.manifest.j2

+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-scheduler
+  namespace: kube-system
+spec:
+  hostNetwork: true
+  containers:
+  - name: kube-scheduler
+    image: {{ hyperkube_image_repo }}:{{ hyperkube_image_tag }}
+    command:
+    - /hyperkube
+    - scheduler
+    - --master=http://127.0.0.1:{{kube_apiserver_insecure_port}}
+    - --v={{ kube_log_level | default('2') }}
+    livenessProbe:
+      httpGet:
+        host: 127.0.0.1
+        path: /healthz
+        port: 10251
+      initialDelaySeconds: 15
+      timeoutSeconds: 1

roles/kubernetes/master/templates/proxy.j2

-###
-# kubernetes proxy config
-
-# default config should be adequate
-
-# Add your own!
-[Service]
-Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}"