diff --git a/contrib/inventory_builder/inventory.py b/contrib/inventory_builder/inventory.py
index 2f02375c6664a6aa22d6a987ffb997bd97c68339..7e0a89f09f044a779895b2d757b1fc45733bbcfb 100644
--- a/contrib/inventory_builder/inventory.py
+++ b/contrib/inventory_builder/inventory.py
@@ -40,7 +40,8 @@ import os
import re
import sys
-ROLES = ['kube-master', 'all', 'k8s-cluster:children', 'kube-node', 'etcd']
+ROLES = ['all', 'kube-master', 'kube-node', 'etcd', 'k8s-cluster:children',
+ 'calico-rr']
PROTECTED_NAMES = ROLES
AVAILABLE_COMMANDS = ['help', 'print_cfg', 'print_ips', 'load']
_boolean_states = {'1': True, 'yes': True, 'true': True, 'on': True,
@@ -51,10 +52,18 @@ def get_var_as_bool(name, default):
value = os.environ.get(name, '')
return _boolean_states.get(value.lower(), default)
+# Configurable as shell vars start
+
CONFIG_FILE = os.environ.get("CONFIG_FILE", "./inventory.cfg")
+# Reconfigures cluster distribution at scale
+SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 50))
+MASSIVE_SCALE_THRESHOLD = int(os.environ.get("SCALE_THRESHOLD", 200))
+
DEBUG = get_var_as_bool("DEBUG", True)
HOST_PREFIX = os.environ.get("HOST_PREFIX", "node")
+# Configurable as shell vars end
+
class KargoInventory(object):
@@ -74,11 +83,16 @@ class KargoInventory(object):
if changed_hosts:
self.hosts = self.build_hostnames(changed_hosts)
self.purge_invalid_hosts(self.hosts.keys(), PROTECTED_NAMES)
- self.set_kube_master(list(self.hosts.keys())[:2])
self.set_all(self.hosts)
self.set_k8s_cluster()
- self.set_kube_node(self.hosts.keys())
self.set_etcd(list(self.hosts.keys())[:3])
+ if len(self.hosts) >= SCALE_THRESHOLD:
+ self.set_kube_master(list(self.hosts.keys())[3:5])
+ else:
+ self.set_kube_master(list(self.hosts.keys())[:2])
+ self.set_kube_node(self.hosts.keys())
+ if len(self.hosts) >= SCALE_THRESHOLD:
+ self.set_calico_rr(list(self.hosts.keys())[:3])
else: # Show help if no options
self.show_help()
sys.exit(0)
@@ -205,8 +219,32 @@ class KargoInventory(object):
self.add_host_to_group('k8s-cluster:children', 'kube-node')
self.add_host_to_group('k8s-cluster:children', 'kube-master')
+ def set_calico_rr(self, hosts):
+ for host in hosts:
+ if host in self.config.items('kube-master'):
+ self.debug("Not adding {0} to calico-rr group because it "
+ "conflicts with kube-master group".format(host))
+ continue
+ if host in self.config.items('kube-node'):
+ self.debug("Not adding {0} to calico-rr group because it "
+ "conflicts with kube-node group".format(host))
+ continue
+ self.add_host_to_group('calico-rr', host)
+
def set_kube_node(self, hosts):
for host in hosts:
+ if len(self.config['all']) >= SCALE_THRESHOLD:
+ if self.config.has_option('etcd', host):
+ self.debug("Not adding {0} to kube-node group because of "
+ "scale deployment and host is in etcd "
+ "group.".format(host))
+ continue
+ if len(self.config['all']) >= MASSIVE_SCALE_THRESHOLD:
+ if self.config.has_option('kube-master', host):
+ self.debug("Not adding {0} to kube-node group because of "
+ "scale deployment and host is in kube-master "
+ "group.".format(host))
+ continue
self.add_host_to_group('kube-node', host)
def set_etcd(self, hosts):
@@ -275,7 +313,15 @@ print_ips - Write a space-delimited list of IPs from "all" group
Advanced usage:
Add another host after initial creation: inventory.py 10.10.1.5
Delete a host: inventory.py -10.10.1.3
-Delete a host by id: inventory.py -node1'''
+Delete a host by id: inventory.py -node1
+
+Configurable env vars:
+DEBUG Enable debug printing. Default: True
+CONFIG_FILE File to write config to Default: ./inventory.cfg
+HOST_PREFIX Host prefix for generated hosts. Default: node
+SCALE_THRESHOLD Separate ETCD role if # of nodes >= 50
+MASSIVE_SCALE_THRESHOLD Separate K8s master and ETCD if # of nodes >= 200
+'''
print(help_text)
def print_config(self):
diff --git a/contrib/inventory_builder/requirements.yml b/contrib/inventory_builder/requirements.yml
deleted file mode 100644
index 5e405b1ed5e57fd103f14df8a257b08ca1a0d071..0000000000000000000000000000000000000000
--- a/contrib/inventory_builder/requirements.yml
+++ /dev/null
@@ -1,48 +0,0 @@
----
-- src: https://gitlab.com/kubespray-ansibl8s/k8s-common.git
- path: roles/apps
- scm: git
-
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-dashboard.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-kubedns.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-elasticsearch.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-redis.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-memcached.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-postgres.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-pgbouncer.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-heapster.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-influxdb.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-kubedash.git
-# path: roles/apps
-# scm: git
-#
-#- src: https://gitlab.com/kubespray-ansibl8s/k8s-kube-logstash.git
-# path: roles/apps
-# scm: git
diff --git a/contrib/inventory_builder/tests/test_inventory.py b/contrib/inventory_builder/tests/test_inventory.py
index 681883772014bbe7db396851bd5ae792f76b6f12..ad393079d91989c4798ad63b224af703ae663635 100644
--- a/contrib/inventory_builder/tests/test_inventory.py
+++ b/contrib/inventory_builder/tests/test_inventory.py
@@ -210,3 +210,31 @@ class TestInventory(unittest.TestCase):
self.inv.set_etcd([host])
self.assertTrue(host in self.inv.config[group])
+
+ def test_scale_scenario_one(self):
+ num_nodes = 50
+ hosts = OrderedDict()
+
+ for hostid in range(1, num_nodes+1):
+ hosts["node" + str(hostid)] = ""
+
+ self.inv.set_all(hosts)
+ self.inv.set_etcd(hosts.keys()[0:3])
+ self.inv.set_kube_master(hosts.keys()[0:2])
+ self.inv.set_kube_node(hosts.keys())
+ for h in range(3):
+ self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])
+
+ def test_scale_scenario_two(self):
+ num_nodes = 500
+ hosts = OrderedDict()
+
+ for hostid in range(1, num_nodes+1):
+ hosts["node" + str(hostid)] = ""
+
+ self.inv.set_all(hosts)
+ self.inv.set_etcd(hosts.keys()[0:3])
+ self.inv.set_kube_master(hosts.keys()[3:5])
+ self.inv.set_kube_node(hosts.keys())
+ for h in range(5):
+ self.assertFalse(hosts.keys()[h] in self.inv.config['kube-node'])
diff --git a/contrib/inventory_builder/tox.ini b/contrib/inventory_builder/tox.ini
index 8ca254295c90991222d2164f2d9a507729a56626..ae675f76bbff9b82271efd541a76afe719a1019d 100644
--- a/contrib/inventory_builder/tox.ini
+++ b/contrib/inventory_builder/tox.ini
@@ -11,7 +11,7 @@ deps =
-r{toxinidir}/test-requirements.txt
setenv = VIRTUAL_ENV={envdir}
passenv = http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
-commands = py.test -vv #{posargs:./tests}
+commands = pytest -vv #{posargs:./tests}
[testenv:pep8]
usedevelop = False
diff --git a/docs/getting-started.md b/docs/getting-started.md
index 906f78c0733ea71edb6d099e2ee248b1e30f8a08..8958d609d2302c7562a5317319085e07283d9414 100644
--- a/docs/getting-started.md
+++ b/docs/getting-started.md
@@ -23,19 +23,17 @@ Building your own inventory
Ansible inventory can be stored in 3 formats: YAML, JSON, or inifile. There is
an example inventory located
-[here](https://github.com/kubernetes-incubator/kargo/blob/master/inventory/inventory.example):
+[here](https://github.com/kubernetes-incubator/kargo/blob/master/inventory/inventory.example).
-```
-cp -r inventory my_inventory
-cp my_inventory/inventory.example my_inventory/inventory.cfg
-# edit the inventory file as needed
-```
-
-Or you can use an
+You can use an
[inventory generator](https://github.com/kubernetes-incubator/kargo/blob/master/contrib/inventory_builder/inventory.py)
to create or modify an Ansible inventory. Currently, it is limited in
functionality and is only use for making a basic Kargo cluster, but it does
-support creating large clusters. For example:
+support creating large clusters. It now supports
+separated ETCD and Kubernetes master roles from node role if the size exceeds a
+certain threshold. Run inventory.py help for more information.
+
+Example inventory generator usage:
```
cp -r inventory my_inventory
diff --git a/docs/large-deployments.md b/docs/large-deployments.md
index 20bc7fefd982730c13addfffdec42425c55f05c3..721064942a129ab1e5833e9041dd48e1b10c1dd5 100644
--- a/docs/large-deployments.md
+++ b/docs/large-deployments.md
@@ -27,5 +27,15 @@ For a large scaled deployments, consider the following configuration changes:
end up with the 'm' skipped for docker as well. This is required as docker does not
understand k8s units well.
+* Add calico-rr nodes if you are deploying with Calico or Canal. Nodes recover
+ from host/network interruption much quicker with calico-rr. Note that
+ calico-rr role must be on a host without kube-master or kube-node role (but
+ etcd role is okay).
+
+* Check out the
+ [Inventory](https://github.com/kubernetes-incubator/kargo/blob/master/docs/getting-started.md#building-your-own-inventory)
+ section of the Getting started guide for tips on creating a large scale
+ Ansible inventory.
+
For example, when deploying 200 nodes, you may want to run ansible with
``--forks=50``, ``--timeout=600`` and define the ``retry_stagger: 60``.