Skip to content
Snippets Groups Projects
Commit f4962051 authored by Smaine Kahlouch's avatar Smaine Kahlouch
Browse files

running kubernetes master processes as pods

parent ef8a46b8
No related branches found
No related tags found
No related merge requests found
Showing
with 238 additions and 342 deletions
---
- name: restart daemons
command: /bin/true
notify:
- reload systemd
- restart reloaded-scheduler
- restart reloaded-controller-manager
- restart reloaded-apiserver
- restart reloaded-proxy
- name: reload systemd
command: systemctl daemon-reload
- name: restart apiserver
command: /bin/true
notify:
- reload systemd
- restart reloaded-apiserver
- name: restart reloaded-apiserver
service:
name: kube-apiserver
state: restarted
- name: restart controller-manager
command: /bin/true
notify:
- reload systemd
- restart reloaded-controller-manager
- name: restart reloaded-controller-manager
service:
name: kube-controller-manager
state: restarted
- name: restart scheduler
- name: restart kubelet
command: /bin/true
notify:
- reload systemd
- restart reloaded-scheduler
- restart reloaded-kubelet
- name: restart reloaded-scheduler
- name: restart reloaded-kubelet
service:
name: kube-scheduler
name: kubelet
state: restarted
- name: restart proxy
......
---
dependencies:
- { role: kubernetes/common }
- { role: etcd }
- { role: kubernetes/node }
---
- name: get the node token values from token files
slurp:
src: "{{ kube_token_dir }}/{{ item }}-{{ inventory_hostname }}.token"
with_items:
- "system:controller_manager"
- "system:scheduler"
- "system:kubectl"
- "system:proxy"
register: tokens
delegate_to: "{{ groups['kube-master'][0] }}"
- name: Set token facts
set_fact:
controller_manager_token: "{{ tokens.results[0].content|b64decode }}"
scheduler_token: "{{ tokens.results[1].content|b64decode }}"
kubectl_token: "{{ tokens.results[2].content|b64decode }}"
proxy_token: "{{ tokens.results[3].content|b64decode }}"
- name: write the config files for api server
template: src=apiserver.j2 dest={{ kube_config_dir }}/apiserver backup=yes
notify:
- restart apiserver
- name: write config file for controller-manager
template: src=controller-manager.j2 dest={{ kube_config_dir }}/controller-manager backup=yes
notify:
- restart controller-manager
- name: write the kubecfg (auth) file for controller-manager
template: src=controller-manager.kubeconfig.j2 dest={{ kube_config_dir }}/controller-manager.kubeconfig backup=yes
notify:
- restart controller-manager
- name: write the config file for scheduler
template: src=scheduler.j2 dest={{ kube_config_dir }}/scheduler backup=yes
notify:
- restart scheduler
- name: write the kubecfg (auth) file for scheduler
template: src=scheduler.kubeconfig.j2 dest={{ kube_config_dir }}/scheduler.kubeconfig backup=yes
notify:
- restart scheduler
- name: write the kubecfg (auth) file for kubectl
template: src=kubectl.kubeconfig.j2 dest={{ kube_config_dir }}/kubectl.kubeconfig backup=yes
- name: Copy kubectl bash completion
copy: src=kubectl_bash_completion.sh dest=/etc/bash_completion.d/kubectl.sh
- name: Create proxy environment vars dir
file: path=/etc/systemd/system/kube-proxy.service.d state=directory
- name: Write proxy config file
template: src=proxy.j2 dest=/etc/systemd/system/kube-proxy.service.d/10-proxy-cluster.conf backup=yes
notify:
- restart proxy
- name: write the kubecfg (auth) file for proxy
template: src=proxy.kubeconfig.j2 dest={{ kube_config_dir }}/proxy.kubeconfig backup=yes
- name: populate users for basic auth in API
lineinfile:
dest: "{{ kube_users_dir }}/known_users.csv"
create: yes
line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
backup: yes
with_dict: "{{ kube_users }}"
notify:
- restart apiserver
- name: Enable controller-manager
service:
name: kube-controller-manager
enabled: yes
state: started
- name: Enable scheduler
service:
name: kube-scheduler
enabled: yes
state: started
- name: Enable kube-proxy
service:
name: kube-proxy
enabled: yes
state: started
- name: Enable apiserver
service:
name: kube-apiserver
enabled: yes
state: started
---
- name: Write kube-apiserver systemd init file
template: src=systemd-init/kube-apiserver.service.j2 dest=/etc/systemd/system/kube-apiserver.service backup=yes
notify: restart apiserver
- name: Write kube-controller-manager systemd init file
template: src=systemd-init/kube-controller-manager.service.j2 dest=/etc/systemd/system/kube-controller-manager.service backup=yes
notify: restart controller-manager
- name: Write kube-scheduler systemd init file
template: src=systemd-init/kube-scheduler.service.j2 dest=/etc/systemd/system/kube-scheduler.service backup=yes
notify: restart scheduler
- name: Write kube-proxy systemd init file
template: src=systemd-init/kube-proxy.service.j2 dest=/etc/systemd/system/kube-proxy.service backup=yes
notify: restart proxy
- name: Install kubernetes binaries
copy:
src={{ local_release_dir }}/kubernetes/bin/{{ item }}
dest={{ bin_dir }}
owner=kube
mode=u+x
with_items:
- kube-apiserver
- kube-controller-manager
- kube-scheduler
- kube-proxy
- kubectl
notify:
- restart daemons
- name: Allow apiserver to bind on both secure and insecure ports
shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver
---
- include: install.yml
- include: config.yml
- name: Install kubectl binary
copy:
src={{ local_release_dir }}/kubernetes/bin/kubectl
dest={{ bin_dir }}
owner=kube
mode=u+x
notify:
- restart daemons
- name: Copy kubectl bash completion
copy:
src: kubectl_bash_completion.sh
dest: /etc/bash_completion.d/kubectl.sh
- name: populate users for basic auth in API
lineinfile:
dest: "{{ kube_users_dir }}/known_users.csv"
create: yes
line: '{{ item.value.pass }},{{ item.key }},{{ item.value.role }}'
backup: yes
with_dict: "{{ kube_users }}"
# Sync masters
- name: synchronize auth directories for masters
synchronize:
src: "{{ item }}"
dest: "{{ kube_config_dir }}"
recursive: yes
delete: yes
rsync_opts: [ '--one-file-system']
with_items:
- "{{ kube_token_dir }}"
- "{{ kube_cert_dir }}"
- "{{ kube_users_dir }}"
delegate_to: "{{ groups['kube-master'][0] }}"
# Write manifests
- name: Write kube-apiserver manifest
template:
src: manifests/kube-apiserver.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-apisever.manifest"
notify:
- restart kubelet
- meta: flush_handlers
- name: wait for the apiserver to be running (pulling image and running container)
wait_for:
port: 8080
- name: install required python module 'httplib2'
apt:
name: "python-httplib2"
state: present
when: inventory_hostname == groups['kube-master'][0]
- name: Create 'kube-system' namespace
uri:
url: http://{{ groups['kube-master'][0]}}:{{ kube_apiserver_insecure_port }}/api/v1/namespaces
method: POST
body: '{"apiVersion":"v1","kind":"Namespace","metadata":{"name":"kube-system"}}'
status_code: 201,409
body_format: json
run_once: yes
when: inventory_hostname == groups['kube-master'][0]
- name: Write kube-controller-manager manifest
template:
src: manifests/kube-controller-manager.manifest.j2
dest: "{{ kube_config_dir }}/kube-controller-manager.manifest"
- name: Write kube-scheduler manifest
template:
src: manifests/kube-scheduler.manifest.j2
dest: "{{ kube_config_dir }}/kube-scheduler.manifest"
- name: Write podmaster manifest
template:
src: manifests/kube-podmaster.manifest.j2
dest: "{{ kube_manifest_dir }}/kube-podmaster.manifest"
###
# kubernetes system config
#
# The following values are used to configure the kube-apiserver
#
# The address on the local server to listen to.
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0"
# The port on the local server to listen on.
KUBE_API_PORT="--insecure-port={{kube_master_insecure_port}} --secure-port={{ kube_master_port }}"
# KUBELET_PORT="--kubelet_port=10250"
# Address range to use for services
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range={{ kube_service_addresses }}"
# Location of the etcd cluster
KUBE_ETCD_SERVERS="--etcd_servers={% for node in groups['etcd'] %}http://{{ node }}:2379{% if not loop.last %},{% endif %}{% endfor %}"
# default admission control policies
KUBE_ADMISSION_CONTROL="--admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
# RUNTIME API CONFIGURATION (e.g. enable extensions)
KUBE_RUNTIME_CONFIG="{% if kube_api_runtime_config is defined %}{% for conf in kube_api_runtime_config %}--runtime-config={{ conf }} {% endfor %}{% endif %}"
# Add you own!
KUBE_API_ARGS="--tls_cert_file={{ kube_cert_dir }}/server.crt --tls_private_key_file={{ kube_cert_dir }}/server.key --client_ca_file={{ kube_cert_dir }}/ca.crt --token_auth_file={{ kube_token_dir }}/known_tokens.csv --basic-auth-file={{ kube_users_dir }}/known_users.csv --service_account_key_file={{ kube_cert_dir }}/server.crt"
###
# The following values are used to configure the kubernetes controller-manager
# defaults from config and apiserver should be adequate
KUBE_CONTROLLER_MANAGER_ARGS="--kubeconfig={{ kube_config_dir }}/controller-manager.kubeconfig --service_account_private_key_file={{ kube_cert_dir }}/server.key --root_ca_file={{ kube_cert_dir }}/ca.crt"
apiVersion: v1
kind: Config
current-context: controller-manager-to-{{ cluster_name }}
preferences: {}
clusters:
- cluster:
certificate-authority: {{ kube_cert_dir }}/ca.crt
server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
name: {{ cluster_name }}
contexts:
- context:
cluster: {{ cluster_name }}
user: controller-manager
name: controller-manager-to-{{ cluster_name }}
users:
- name: controller-manager
user:
token: {{ controller_manager_token }}
......@@ -4,8 +4,8 @@ current-context: kubectl-to-{{ cluster_name }}
preferences: {}
clusters:
- cluster:
certificate-authority-data: {{ kube_ca_cert|b64encode }}
server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
certificate-authority-data: {{ kube_node_cert|b64encode }}
server: https://{{ groups['kube-master'][0] }}:{{ kube_apiserver_port }}
name: {{ cluster_name }}
contexts:
- context:
......
apiVersion: v1
kind: Pod
metadata:
name: kube-apiserver
spec:
hostNetwork: true
containers:
- name: kube-apiserver
image: {{ hyperkube_image.name }}:{{ hyperkube_image.tag }}
command:
- /hyperkube
- apiserver
- --insecure-bind-address=0.0.0.0
- --etcd-servers=http://{{ hostvars[inventory_hostname]['ip'] | default( ansible_default_ipv4.address) }}:2379
- --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota
- --service-cluster-ip-range={{ kube_service_addresses }}
- --client-ca-file={{ kube_cert_dir }}/ca.pem
- --basic-auth-file={{ kube_users_dir }}/known_users.csv
- --tls-cert-file={{ kube_cert_dir }}/apiserver.pem
- --tls-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
- --service-account-key-file={{ kube_cert_dir }}/apiserver-key.pem
- --secure-port={{ kube_apiserver_port }}
- --token-auth-file={{ kube_token_dir }}/known_tokens.csv
- --v={{ kube_log_level | default('2') }}
- --allow-privileged=true
ports:
- containerPort: {{ kube_apiserver_port }}
hostPort: {{ kube_apiserver_port }}
name: https
- containerPort: {{ kube_apiserver_insecure_port }}
hostPort: {{ kube_apiserver_insecure_port }}
name: local
volumeMounts:
- mountPath: {{ kube_config_dir }}
name: kubernetes-config
readOnly: true
- mountPath: /etc/ssl/certs
name: ssl-certs-host
readOnly: true
volumes:
- hostPath:
path: {{ kube_config_dir }}
name: kubernetes-config
- hostPath:
path: /usr/share/ca-certificates
name: ssl-certs-host
apiVersion: v1
kind: Pod
metadata:
name: kube-controller-manager
namespace: kube-system
spec:
hostNetwork: true
containers:
- name: kube-controller-manager
image: {{ hyperkube_image.name }}:{{ hyperkube_image.tag }}
command:
- /hyperkube
- controller-manager
- --master=http://127.0.0.1:8080
- --service-account-private-key-file={{ kube_cert_dir }}/apiserver-key.pem
- --root-ca-file={{ kube_cert_dir }}/ca.pem
- --v={{ kube_log_level | default('2') }}
livenessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: 10252
initialDelaySeconds: 15
timeoutSeconds: 1
volumeMounts:
- mountPath: {{ kube_cert_dir }}
name: ssl-certs-kubernetes
readOnly: true
- mountPath: /etc/ssl/certs
name: ssl-certs-host
readOnly: true
volumes:
- hostPath:
path: {{ kube_cert_dir }}
name: ssl-certs-kubernetes
- hostPath:
path: /usr/share/ca-certificates
name: ssl-certs-host
apiVersion: v1
kind: Pod
metadata:
name: kube-podmaster
namespace: kube-system
spec:
hostNetwork: true
containers:
- name: scheduler-elector
image: gcr.io/google_containers/podmaster:1.1
command:
- /podmaster
- --etcd-servers=http://127.0.0.1:2379
- --key=scheduler
- --source-file={{ kube_config_dir}}/kube-scheduler.manifest
- --dest-file={{ kube_manifest_dir }}/kube-scheduler.manifest
volumeMounts:
- mountPath: {{ kube_config_dir }}
name: manifest-src
readOnly: true
- mountPath: {{ kube_manifest_dir }}
name: manifest-dst
- name: controller-manager-elector
image: gcr.io/google_containers/podmaster:1.1
command:
- /podmaster
- --etcd-servers=http://127.0.0.1:2379
- --key=controller
- --source-file={{ kube_config_dir }}/kube-controller-manager.manifest
- --dest-file={{ kube_manifest_dir }}/kube-controller-manager.manifest
terminationMessagePath: /dev/termination-log
volumeMounts:
- mountPath: {{ kube_config_dir }}
name: manifest-src
readOnly: true
- mountPath: {{ kube_manifest_dir }}
name: manifest-dst
volumes:
- hostPath:
path: {{ kube_config_dir }}
name: manifest-src
- hostPath:
path: {{ kube_manifest_dir }}
name: manifest-dst
apiVersion: v1
kind: Pod
metadata:
name: kube-scheduler
namespace: kube-system
spec:
hostNetwork: true
containers:
- name: kube-scheduler
image: {{ hyperkube_image.name }}:{{ hyperkube_image.tag }}
command:
- /hyperkube
- scheduler
- --master=http://127.0.0.1:8080
- --v={{ kube_log_level | default('2') }}
livenessProbe:
httpGet:
host: 127.0.0.1
path: /healthz
port: 10251
initialDelaySeconds: 15
timeoutSeconds: 1
###
# kubernetes proxy config
# default config should be adequate
# Add your own!
[Service]
Environment="KUBE_PROXY_ARGS=--kubeconfig={{ kube_config_dir }}/proxy.kubeconfig --proxy-mode={{kube_proxy_mode}}"
apiVersion: v1
kind: Config
current-context: proxy-to-{{ cluster_name }}
preferences: {}
contexts:
- context:
cluster: {{ cluster_name }}
user: proxy
name: proxy-to-{{ cluster_name }}
clusters:
- cluster:
certificate-authority: {{ kube_cert_dir }}/ca.crt
server: http://{{ groups['kube-master'][0] }}:{{kube_master_insecure_port}}
name: {{ cluster_name }}
users:
- name: proxy
user:
token: {{ proxy_token }}
###
# kubernetes scheduler config
# default config should be adequate
# Add your own!
KUBE_SCHEDULER_ARGS="--kubeconfig={{ kube_config_dir }}/scheduler.kubeconfig"
apiVersion: v1
kind: Config
current-context: scheduler-to-{{ cluster_name }}
preferences: {}
clusters:
- cluster:
certificate-authority: {{ kube_cert_dir }}/ca.crt
server: https://{{ groups['kube-master'][0] }}:{{ kube_master_port }}
name: {{ cluster_name }}
contexts:
- context:
cluster: {{ cluster_name }}
user: scheduler
name: scheduler-to-{{ cluster_name }}
users:
- name: scheduler
user:
token: {{ scheduler_token }}
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Requires=etcd2.service
After=etcd2.service
[Service]
EnvironmentFile=/etc/network-environment
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/apiserver
User=kube
ExecStart={{ bin_dir }}/kube-apiserver \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_ETCD_SERVERS \
$KUBE_API_ADDRESS \
$KUBE_API_PORT \
$KUBELET_PORT \
$KUBE_ALLOW_PRIV \
$KUBE_SERVICE_ADDRESSES \
$KUBE_ADMISSION_CONTROL \
$KUBE_RUNTIME_CONFIG \
$KUBE_API_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
Requires=etcd2.service
After=etcd2.service
[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/controller-manager
User=kube
ExecStart={{ bin_dir }}/kube-controller-manager \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
{% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
After=docker.service calico-node.service
{% else %}
After=docker.service
{% endif %}
[Service]
EnvironmentFile=/etc/kubernetes/config
EnvironmentFile=/etc/network-environment
ExecStart={{ bin_dir }}/kube-proxy \
$KUBE_LOGTOSTDERR \
$KUBE_LOG_LEVEL \
$KUBE_MASTER \
$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment