Merge pull request #132 from Smana/net_plugins_distinct_roles

split network plugins into distinct roles

README.md

@@ -25,7 +25,7 @@ in order to avoid any issue during deployment you should disable your firewall
 ### Components
 * [kubernetes](https://github.com/kubernetes/kubernetes/releases) v1.1.4
 * [etcd](https://github.com/coreos/etcd/releases) v2.2.4
-* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.14.0
+* [calicoctl](https://github.com/projectcalico/calico-docker/releases) v0.16.0
 * [flanneld](https://github.com/coreos/flannel/releases) v0.5.5
 * [docker](https://www.docker.com/) v1.9.1
 
@@ -107,21 +107,20 @@ kube-master
 ### Playbook
 ```
 ---
-
 - hosts: k8s-cluster
   roles:
+    - { role: adduser, tags: adduser }
     - { role: download, tags: download }
     - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: etcd, tags: etcd }
     - { role: docker, tags: docker }
     - { role: kubernetes/node, tags: node }
-    - { role: etcd, tags: etcd }
+    - { role: network_plugin, tags: network }
     - { role: dnsmasq, tags: dnsmasq }
-    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
 
 - hosts: kube-master
   roles:
     - { role: kubernetes/master, tags: master }
-
 ```
 
 ### Run
@@ -143,14 +142,14 @@ the server address has to be present on both groups 'kube-master' and 'kube-node
 In order to do so, some variables have to be used '**loadbalancer_apiserver**' and '**apiserver_loadbalancer_domain_name**'
 
 
-### Network Overlay
+### Network Plugin
 You can choose between 2 network plugins. Only one must be chosen.
 
 * **flannel**: gre/vxlan (layer 2) networking. ([official docs](https://github.com/coreos/flannel))
 
 * **calico**: bgp (layer 3) networking. ([official docs](http://docs.projectcalico.org/en/0.13/))
 
-The choice is defined with the variable '**kube_network_plugin**'
+The choice is defined with the variable **kube_network_plugin**
 
 
 ### Check cluster status

apps.yml

@@ -9,7 +9,6 @@
     - { role: apps/k8s-elasticsearch, tags: 'elasticsearch' }
     - { role: apps/k8s-memcached, tags: 'memcached' }
     - { role: apps/k8s-redis, tags: 'redis' }
-    - { role: apps/k8s-mongodb-simple, tags: 'mongodb-simple' }
 
     # Msg Broker
     - { role: apps/k8s-rabbitmq, tags: 'rabbitmq' }
@@ -28,6 +27,3 @@
 
     # ETCD
     - { role: apps/k8s-etcd, tags: 'etcd'}
-
-    # Chat Apps
-    - { role: apps/k8s-rocketchat, tags: 'rocketchat'}
\ No newline at end of file

cluster.yml

@@ -4,11 +4,11 @@
     - { role: adduser, tags: adduser }
     - { role: download, tags: download }
     - { role: kubernetes/preinstall, tags: preinstall }
+    - { role: etcd, tags: etcd }
     - { role: docker, tags: docker }
     - { role: kubernetes/node, tags: node }
-    - { role: etcd, tags: etcd }
+    - { role: network_plugin, tags: network }
     - { role: dnsmasq, tags: dnsmasq }
-    - { role: network_plugin, tags: ['calico', 'flannel', 'network'] }
 
 - hosts: kube-master
   roles:

inventory/group_vars/all.yml

@@ -24,9 +24,6 @@ kube_users:
 # Kubernetes cluster name, also will be used as DNS domain
 cluster_name: cluster.local
 
-# set this variable to calico if needed. keep it empty if flannel is used
-kube_network_plugin: calico
-
 # For some environments, each node has a pubilcally accessible
 # address and an address it should bind services to.  These are
 # really inventory level variables, but described here for consistency.
@@ -49,6 +46,9 @@ kube_network_plugin: calico
 # but don't know about that address themselves.
 # access_ip: 1.1.1.1
 
+# Choose network plugin (calico or flannel)
+kube_network_plugin: calico
+
 # Kubernetes internal network for services, unused block of space.
 kube_service_addresses: 10.233.0.0/18
 

roles/docker/tasks/main.yml

@@ -41,7 +41,7 @@
   action: "{{ docker_package_info.pkg_mgr }}"
   args:
     pkg: "{{item}}"
-    state: latest
+    state: present
   with_items: docker_package_info.pkgs
   when: docker_package_info.pkgs|length > 0
 

roles/docker/vars/debian.yml

 docker_kernel_min_version: '3.2'
+docker_version: 1.9.1-0~{{ ansible_distribution_release|lower }}
 
 docker_package_info:
   pkg_mgr: apt
   pkgs:
-    - docker-engine
+    - docker-engine={{ docker_version }}
 
 docker_repo_key_info:
   pkg_key: apt_key

roles/docker/vars/ubuntu.yml

+docker_kernel_min_version: '3.2'
+docker_version: 1.9.0-0~{{ ansible_distribution_release }}
+
+docker_package_info:
+  pkg_mgr: apt
+  pkgs:
+    - docker-engine={{ docker_version }}
+
+docker_repo_key_info:
+  pkg_key: apt_key
+  keyserver: hkp://p80.pool.sks-keyservers.net:80
+  repo_keys:
+    - 58118E89F3A912897C070ADBF76221572C52609D
+
+docker_repo_info:
+  pkg_repo: apt_repository
+  repos:
+    - >
+       deb https://apt.dockerproject.org/repo
+       {{ ansible_distribution|lower }}-{{ ansible_distribution_release|lower }}
+       main

roles/download/defaults/main.yml

@@ -4,18 +4,20 @@ local_release_dir: /tmp
 # Versions
 kube_version: v1.1.4
 etcd_version: v2.2.4
-calico_version: v0.14.0
-calico_plugin_version: v0.7.0
+calico_version: v0.16.0
+calico_cni_version: v1.0.0
 
 # Download URL's
 kube_download_url: "https://storage.googleapis.com/kubernetes-release/release/{{ kube_version }}/bin/linux/amd64"
 etcd_download_url: "https://github.com/coreos/etcd/releases/download/{{ etcd_version }}/etcd-{{ etcd_version }}-linux-amd64.tar.gz"
 calico_download_url: "https://github.com/Metaswitch/calico-docker/releases/download/{{calico_version}}/calicoctl"
-calico_plugin_download_url: "https://github.com/projectcalico/calico-kubernetes/releases/download/{{calico_plugin_version}}/calico_kubernetes"
+calico_cni_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico"
+calico_cni_ipam_download_url: "https://github.com/projectcalico/calico-cni/releases/download/{{calico_cni_version}}/calico-ipam"
 
 # Checksums
-calico_checksum: "f251d7a8583233906aa6d059447c1e4fb32bf1369a51fdf96a68d50466d6a69c"
-calico_plugin_checksum: "032f582f5eeec6fb26191d2fbcbf8bca4da3b14abb579db7baa7b3504d4dffec"
+calico_checksum: "cfbbcad4b3b7d79be9a25bcdc153ec1d139eecd54840914a363b0710eebc5c51"
+calico_cni_checksum: "cfbb95d4416cb65845a188f3bd991fff232bd5ce3463b2919d586ab77967aecd"
+calico_cni_ipam_checksum: "93ebf8756b26314e1e3f612f1e824418cbb0a8df2942664422e697bcb109fbb2"
 etcd_checksum: "6c4e5cdeaaac1a70b8f06b5dd6b82c37ff19993c9bca81248975610e555c4b9b"
 kubectl_checksum: "873ba19926d17a3287dc8639ea1434fe3cd0cb4e61d82101ba754922cfc7a633"
 kubelet_checksum: "f2d1eae3fa6e304f6cbc9b2621e4b86fc3bcb4e74a15d35f58bf00e45c706e0a"
@@ -29,10 +31,17 @@ downloads:
     owner: "root"
     mode: "0755"
 
-  - name: calico-plugin
+  - name: calico-cni-plugin
     dest: calico/bin/calico
-    sha256: "{{ calico_plugin_checksum }}"
-    url: "{{ calico_plugin_download_url }}"
+    sha256: "{{ calico_cni_checksum }}"
+    url: "{{ calico_cni_download_url }}"
+    owner: "root"
+    mode: "0755"
+
+  - name: calico-cni-plugin-ipam
+    dest: calico/bin/calico-ipam
+    sha256: "{{ calico_cni_ipam_checksum }}"
+    url: "{{ calico_cni_ipam_download_url }}"
     owner: "root"
     mode: "0755"
 

roles/kubernetes/master/tasks/gen_kube_tokens.yml

 ---
+- name: tokens | copy the token gen script
+  copy:
+    src=kube-gen-token.sh
+    dest={{ kube_script_dir }}
+    mode=u+x
+  when: inventory_hostname == groups['kube-master'][0]
+
 - name: tokens | generate tokens for master components
   command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
   environment:

roles/kubernetes/master/tasks/main.yml

@@ -69,11 +69,6 @@
   shell: setcap cap_net_bind_service+ep {{ bin_dir }}/kube-apiserver
   changed_when: false
 
-- name: Restart apiserver
-  command: "/bin/true"
-  notify: restart kube-apiserver
-  when: is_gentoken_calico|default(false)
-
 - meta: flush_handlers
 
 - include: start.yml

roles/kubernetes/node/handlers/main.yml

@@ -9,10 +9,6 @@
     - reload systemd
     - reload kubelet
 
-- name: set is_gentoken_calico fact
-  set_fact:
-    is_gentoken_calico: true
-
 - name: reload kubelet
   service:
     name: kubelet

roles/kubernetes/node/tasks/gen_calico_tokens.yml

----
-- name: tokens | copy the token gen script
-  copy:
-    src=kube-gen-token.sh
-    dest={{ kube_script_dir }}
-    mode=u+x
-  when: inventory_hostname == groups['kube-master'][0]
-
-- name: tokens | generate tokens for calico
-  command: "{{ kube_script_dir }}/kube-gen-token.sh {{ item[0] }}-{{ item[1] }}"
-  environment:
-    TOKEN_DIR: "{{ kube_token_dir }}"
-  with_nested:
-    - [ "system:calico" ]
-    - "{{ groups['k8s-cluster'] }}"
-  register: gentoken_calico
-  changed_when: "'Added' in gentoken_calico.stdout"
-  when: kube_network_plugin == "calico"
-  delegate_to: "{{ groups['kube-master'][0] }}"
-  notify: set is_gentoken_calico fact
-
-- name: tokens | get the calico token values
-  slurp:
-    src: "{{ kube_token_dir }}/system:calico-{{ inventory_hostname }}.token"
-  register: calico_token
-  when: kube_network_plugin == "calico"
-  delegate_to: "{{ groups['kube-master'][0] }}"

roles/kubernetes/node/tasks/main.yml

 ---
-- name: Create kubernetes config directory
-  file:
-    path: "{{ kube_config_dir }}"
-    state: directory
-    owner: kube
-
-- name: Create kubernetes script directory
-  file:
-    path: "{{ kube_script_dir }}"
-    state: directory
-    owner: kube
-
-- name: Create kubernetes manifests directory
-  file:
-    path: "{{ kube_manifest_dir }}"
-    state: directory
-    owner: kube
-
-- name: Create kubernetes logs directory
-  file:
-    path: "{{ kube_log_dir }}"
-    state: directory
+- name: Write Calico cni config
+  template:
+    src: "cni-calico.conf.j2"
+    dest: "/etc/cni/net.d/10-calico.conf"
     owner: kube
-  when: init_system == "sysvinit"
+  when: kube_network_plugin == "calico"
 
 - include: secrets.yml
-  tags:
-    - secrets
 
 - include: install.yml
 

roles/kubernetes/node/tasks/secrets.yml

@@ -16,8 +16,6 @@
 - include: gen_certs.yml
   when: inventory_hostname == groups['kube-master'][0]
 
-- include: gen_calico_tokens.yml
-
 # Sync certs between nodes
 - name: Secrets | create user
   user:

roles/kubernetes/node/templates/cni-calico.conf.j2

+{
+  "name": "calico-k8s-network",
+  "type": "calico",
+  "etcd_authority": "127.0.0.1:2379",
+  "log_level": "info",
+  "ipam": {
+    "type": "calico-ipam"
+  }
+}

roles/kubernetes/node/templates/kubelet.j2

@@ -24,7 +24,7 @@ KUBELET_ARGS="--cluster_dns={{ dns_server }} --cluster_domain={{ dns_domain }} -
 KUBELET_ARGS="--kubeconfig={{ kube_config_dir}}/kubelet.kubeconfig --config={{ kube_manifest_dir }}"
 {% endif %}
 {% if kube_network_plugin is defined and kube_network_plugin == "calico" %}
-KUBELET_NETWORK_PLUGIN="--network_plugin={{ kube_network_plugin }}"
+KUBELET_NETWORK_PLUGIN="--network_plugin=cni --network-plugin-dir=/etc/cni/net.d"
 {% endif %}
 # Should this cluster be allowed to run privileged docker containers
 KUBE_ALLOW_PRIV="--allow_privileged=true"

roles/kubernetes/preinstall/tasks/main.yml

@@ -33,6 +33,41 @@
   always_run: True
   tags: always
 
+- name: Create kubernetes config directory
+  file:
+    path: "{{ kube_config_dir }}"
+    state: directory
+    owner: kube
+
+- name: Create kubernetes script directory
+  file:
+    path: "{{ kube_script_dir }}"
+    state: directory
+    owner: kube
+
+- name: Create kubernetes manifests directory
+  file:
+    path: "{{ kube_manifest_dir }}"
+    state: directory
+    owner: kube
+
+- name: Create kubernetes logs directory
+  file:
+    path: "{{ kube_log_dir }}"
+    state: directory
+    owner: kube
+  when: init_system == "sysvinit"
+
+- name: Create cni directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: kube
+  with_items:
+    - "/etc/cni/net.d"
+    - "/opt/cni/bin"
+  when: kube_network_plugin == "calico"
+
 - name: Update package management cache (APT)
   apt: update_cache=yes
   when: ansible_pkg_mgr == 'apt'

roles/network_plugin/calico/defaults/main.yml

+---
+# cloud_provider: no

roles/network_plugin/calico/handlers/main.yml

+---
+- name: restart calico-node
+  command: /bin/true
+  notify:
+    - reload systemd
+    - reload calico-node
+
+- name : reload systemd
+  shell: systemctl daemon-reload
+  when: init_system == "systemd"
+
+- name: reload calico-node
+  service:
+    name: calico-node
+    state: restarted