diff --git a/contrib/terraform/openstack/README.md b/contrib/terraform/openstack/README.md
index 09c7c4ddb68efb3aa3bc9bd154afb5a4fed92c2e..be74b7c4e24d0205c290ef589e47c070b6b861a4 100644
--- a/contrib/terraform/openstack/README.md
+++ b/contrib/terraform/openstack/README.md
@@ -264,6 +264,107 @@ For your cluster, edit `inventory/$CLUSTER/cluster.tfvars`.
|`etcd_root_volume_size_in_gb` | Size of the root volume for etcd nodes, 0 to use ephemeral storage |
|`bastion_root_volume_size_in_gb` | Size of the root volume for bastions, 0 to use ephemeral storage |
|`use_server_group` | Create and use openstack nova servergroups, default: false |
+|`k8s_nodes` | Map containing worker node definition, see explanation below |
+
+##### k8s_nodes
+Allows a custom defintion of worker nodes giving the operator full control over individual node flavor and
+availability zone placement. To enable the use of this mode set the `number_of_k8s_nodes` and
+`number_of_k8s_nodes_no_floating_ip` variables to 0. Then define your desired worker node configuration
+using the `k8s_nodes` variable.
+
+For example:
+```
+k8s_nodes = {
+ "1" = {
+ "az" = "sto1"
+ "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
+ "floating_ip" = true
+ },
+ "2" = {
+ "az" = "sto2"
+ "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
+ "floating_ip" = true
+ },
+ "3" = {
+ "az" = "sto3"
+ "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
+ "floating_ip" = true
+ }
+}
+```
+
+Would result in the same configuration as:
+```
+number_of_k8s_nodes = 3
+flavor_k8s_node = "83d8b44a-26a0-4f02-a981-079446926445"
+az_list = ["sto1", "sto2", "sto3"]
+```
+
+And:
+```
+k8s_nodes = {
+ "ing-1" = {
+ "az" = "sto1"
+ "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
+ "floating_ip" = true
+ },
+ "ing-2" = {
+ "az" = "sto2"
+ "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
+ "floating_ip" = true
+ },
+ "ing-3" = {
+ "az" = "sto3"
+ "flavor" = "83d8b44a-26a0-4f02-a981-079446926445"
+ "floating_ip" = true
+ },
+ "big-1" = {
+ "az" = "sto1"
+ "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b"
+ "floating_ip" = false
+ },
+ "big-2" = {
+ "az" = "sto2"
+ "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b"
+ "floating_ip" = false
+ },
+ "big-3" = {
+ "az" = "sto3"
+ "flavor" = "3f73fc93-ec61-4808-88df-2580d94c1a9b"
+ "floating_ip" = false
+ },
+ "small-1" = {
+ "az" = "sto1"
+ "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e"
+ "floating_ip" = false
+ },
+ "small-2" = {
+ "az" = "sto2"
+ "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e"
+ "floating_ip" = false
+ },
+ "small-3" = {
+ "az" = "sto3"
+ "flavor" = "7a6a998f-ac7f-4fb8-a534-2175b254f75e"
+ "floating_ip" = false
+ }
+}
+```
+
+Would result in three nodes in each availability zone each with their own separate naming,
+flavor and floating ip configuration.
+
+The "schema":
+```
+k8s_nodes = {
+ "key | node name suffix, must be unique" = {
+ "az" = string
+ "flavor" = string
+ "floating_ip" = bool
+ },
+}
+```
+All values are required.
#### Terraform state files
@@ -504,3 +605,81 @@ $ ansible-playbook --become -i inventory/$CLUSTER/hosts ./contrib/network-storag
## What's next
Try out your new Kubernetes cluster with the [Hello Kubernetes service](https://kubernetes.io/docs/tasks/access-application-cluster/service-access-application-cluster/).
+
+## Appendix
+
+### Migration from `number_of_k8s_nodes*` to `k8s_nodes`
+If you currently have a cluster defined using the `number_of_k8s_nodes*` variables and wish
+to migrate to the `k8s_nodes` style you can do it like so:
+
+```ShellSession
+$ terraform state list
+module.compute.data.openstack_images_image_v2.gfs_image
+module.compute.data.openstack_images_image_v2.vm_image
+module.compute.openstack_compute_floatingip_associate_v2.k8s_master[0]
+module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]
+module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]
+module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]
+module.compute.openstack_compute_instance_v2.k8s_master[0]
+module.compute.openstack_compute_instance_v2.k8s_node[0]
+module.compute.openstack_compute_instance_v2.k8s_node[1]
+module.compute.openstack_compute_instance_v2.k8s_node[2]
+module.compute.openstack_compute_keypair_v2.k8s
+module.compute.openstack_compute_servergroup_v2.k8s_etcd[0]
+module.compute.openstack_compute_servergroup_v2.k8s_master[0]
+module.compute.openstack_compute_servergroup_v2.k8s_node[0]
+module.compute.openstack_networking_secgroup_rule_v2.bastion[0]
+module.compute.openstack_networking_secgroup_rule_v2.egress[0]
+module.compute.openstack_networking_secgroup_rule_v2.k8s
+module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[0]
+module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[1]
+module.compute.openstack_networking_secgroup_rule_v2.k8s_allowed_remote_ips[2]
+module.compute.openstack_networking_secgroup_rule_v2.k8s_master[0]
+module.compute.openstack_networking_secgroup_rule_v2.worker[0]
+module.compute.openstack_networking_secgroup_rule_v2.worker[1]
+module.compute.openstack_networking_secgroup_rule_v2.worker[2]
+module.compute.openstack_networking_secgroup_rule_v2.worker[3]
+module.compute.openstack_networking_secgroup_rule_v2.worker[4]
+module.compute.openstack_networking_secgroup_v2.bastion[0]
+module.compute.openstack_networking_secgroup_v2.k8s
+module.compute.openstack_networking_secgroup_v2.k8s_master
+module.compute.openstack_networking_secgroup_v2.worker
+module.ips.null_resource.dummy_dependency
+module.ips.openstack_networking_floatingip_v2.k8s_master[0]
+module.ips.openstack_networking_floatingip_v2.k8s_node[0]
+module.ips.openstack_networking_floatingip_v2.k8s_node[1]
+module.ips.openstack_networking_floatingip_v2.k8s_node[2]
+module.network.openstack_networking_network_v2.k8s[0]
+module.network.openstack_networking_router_interface_v2.k8s[0]
+module.network.openstack_networking_router_v2.k8s[0]
+module.network.openstack_networking_subnet_v2.k8s[0]
+$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["1"]'
+Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[0]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"1\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["2"]'
+Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[1]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"2\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]' 'module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes["3"]'
+Move "module.compute.openstack_compute_floatingip_associate_v2.k8s_node[2]" to "module.compute.openstack_compute_floatingip_associate_v2.k8s_nodes[\"3\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[0]' 'module.compute.openstack_compute_instance_v2.k8s_node["1"]'
+Move "module.compute.openstack_compute_instance_v2.k8s_node[0]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"1\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[1]' 'module.compute.openstack_compute_instance_v2.k8s_node["2"]'
+Move "module.compute.openstack_compute_instance_v2.k8s_node[1]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"2\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.compute.openstack_compute_instance_v2.k8s_node[2]' 'module.compute.openstack_compute_instance_v2.k8s_node["3"]'
+Move "module.compute.openstack_compute_instance_v2.k8s_node[2]" to "module.compute.openstack_compute_instance_v2.k8s_node[\"3\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[0]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["1"]'
+Move "module.ips.openstack_networking_floatingip_v2.k8s_node[0]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"1\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[1]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["2"]'
+Move "module.ips.openstack_networking_floatingip_v2.k8s_node[1]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"2\"]"
+Successfully moved 1 object(s).
+$ terraform state mv 'module.ips.openstack_networking_floatingip_v2.k8s_node[2]' 'module.ips.openstack_networking_floatingip_v2.k8s_node["3"]'
+Move "module.ips.openstack_networking_floatingip_v2.k8s_node[2]" to "module.ips.openstack_networking_floatingip_v2.k8s_node[\"3\"]"
+Successfully moved 1 object(s).
+```
+
+Of course for nodes without floating ips those steps can be omitted.
diff --git a/contrib/terraform/openstack/kubespray.tf b/contrib/terraform/openstack/kubespray.tf
index 746b5a550e437e09a13eb8e8b73b2dab2322638d..511027479cac1f45d4245a7576ff088abb315178 100644
--- a/contrib/terraform/openstack/kubespray.tf
+++ b/contrib/terraform/openstack/kubespray.tf
@@ -26,6 +26,7 @@ module "ips" {
external_net = "${var.external_net}"
network_name = "${var.network_name}"
router_id = "${module.network.router_id}"
+ k8s_nodes = "${var.k8s_nodes}"
}
module "compute" {
@@ -43,6 +44,7 @@ module "compute" {
number_of_bastions = "${var.number_of_bastions}"
number_of_k8s_nodes_no_floating_ip = "${var.number_of_k8s_nodes_no_floating_ip}"
number_of_gfs_nodes_no_floating_ip = "${var.number_of_gfs_nodes_no_floating_ip}"
+ k8s_nodes = "${var.k8s_nodes}"
bastion_root_volume_size_in_gb = "${var.bastion_root_volume_size_in_gb}"
etcd_root_volume_size_in_gb = "${var.etcd_root_volume_size_in_gb}"
master_root_volume_size_in_gb = "${var.master_root_volume_size_in_gb}"
@@ -63,6 +65,7 @@ module "compute" {
k8s_master_fips = "${module.ips.k8s_master_fips}"
k8s_master_no_etcd_fips = "${module.ips.k8s_master_no_etcd_fips}"
k8s_node_fips = "${module.ips.k8s_node_fips}"
+ k8s_nodes_fips = "${module.ips.k8s_nodes_fips}"
bastion_fips = "${module.ips.bastion_fips}"
bastion_allowed_remote_ips = "${var.bastion_allowed_remote_ips}"
master_allowed_remote_ips = "${var.master_allowed_remote_ips}"
@@ -95,7 +98,7 @@ output "k8s_master_fips" {
}
output "k8s_node_fips" {
- value = "${module.ips.k8s_node_fips}"
+ value = "${var.number_of_k8s_nodes > 0 ? module.ips.k8s_node_fips : [for key, value in module.ips.k8s_nodes_fips : value.address]}"
}
output "bastion_fips" {
diff --git a/contrib/terraform/openstack/modules/compute/main.tf b/contrib/terraform/openstack/modules/compute/main.tf
index ad6f0dc63d4a3353488a03571bebf60a96694bbc..8be6346c45c55fb409309794ab35ecbeb169f32a 100644
--- a/contrib/terraform/openstack/modules/compute/main.tf
+++ b/contrib/terraform/openstack/modules/compute/main.tf
@@ -465,6 +465,53 @@ resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
}
}
+resource "openstack_compute_instance_v2" "k8s_nodes" {
+ for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? var.k8s_nodes : {}
+ name = "${var.cluster_name}-k8s-node-${each.key}"
+ availability_zone = "${each.value.az}"
+ image_name = "${var.image}"
+ flavor_id = "${each.value.flavor}"
+ key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+
+ dynamic "block_device" {
+ for_each = var.node_root_volume_size_in_gb > 0 ? [var.image] : []
+ content {
+ uuid = "${data.openstack_images_image_v2.vm_image.id}"
+ source_type = "image"
+ volume_size = "${var.node_root_volume_size_in_gb}"
+ boot_index = 0
+ destination_type = "volume"
+ delete_on_termination = true
+ }
+ }
+
+ network {
+ name = "${var.network_name}"
+ }
+
+ security_groups = ["${openstack_networking_secgroup_v2.k8s.name}",
+ "${openstack_networking_secgroup_v2.worker.name}",
+ ]
+
+ dynamic "scheduler_hints" {
+ for_each = var.use_server_groups ? [openstack_compute_servergroup_v2.k8s_node[0]] : []
+ content {
+ group = "${openstack_compute_servergroup_v2.k8s_node[0].id}"
+ }
+ }
+
+ metadata = {
+ ssh_user = "${var.ssh_user}"
+ kubespray_groups = "kube-node,k8s-cluster,%{if each.value.floating_ip == false}no-floating,%{endif}${var.supplementary_node_groups}"
+ depends_on = "${var.network_id}"
+ use_access_ip = "${var.use_access_ip}"
+ }
+
+ provisioner "local-exec" {
+ command = "%{if each.value.floating_ip}sed s/USER/${var.ssh_user}/ ../../contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${element(concat(var.bastion_fips, [for key, value in var.k8s_nodes_fips : value.address]), 0)}/ > group_vars/no-floating.yml%{else}true%{endif}"
+ }
+}
+
resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
name = "${var.cluster_name}-gfs-node-nf-${count.index + 1}"
count = "${var.number_of_gfs_nodes_no_floating_ip}"
@@ -530,7 +577,14 @@ resource "openstack_compute_floatingip_associate_v2" "k8s_master_no_etcd" {
resource "openstack_compute_floatingip_associate_v2" "k8s_node" {
count = "${var.node_root_volume_size_in_gb == 0 ? var.number_of_k8s_nodes : 0}"
floating_ip = "${var.k8s_node_fips[count.index]}"
- instance_id = "${element(openstack_compute_instance_v2.k8s_node.*.id, count.index)}"
+ instance_id = "${element(openstack_compute_instance_v2.k8s_node[*].id, count.index)}"
+ wait_until_associated = "${var.wait_for_floatingip}"
+}
+
+resource "openstack_compute_floatingip_associate_v2" "k8s_nodes" {
+ for_each = var.number_of_k8s_nodes == 0 && var.number_of_k8s_nodes_no_floating_ip == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {}
+ floating_ip = "${var.k8s_nodes_fips[each.key].address}"
+ instance_id = "${openstack_compute_instance_v2.k8s_nodes[each.key].id}"
wait_until_associated = "${var.wait_for_floatingip}"
}
@@ -545,4 +599,4 @@ resource "openstack_compute_volume_attach_v2" "glusterfs_volume" {
count = "${var.gfs_root_volume_size_in_gb == 0 ? var.number_of_gfs_nodes_no_floating_ip : 0}"
instance_id = "${element(openstack_compute_instance_v2.glusterfs_node_no_floating_ip.*.id, count.index)}"
volume_id = "${element(openstack_blockstorage_volume_v2.glusterfs_volume.*.id, count.index)}"
-}
\ No newline at end of file
+}
diff --git a/contrib/terraform/openstack/modules/compute/variables.tf b/contrib/terraform/openstack/modules/compute/variables.tf
index 4fab83c5736ca1f060ea80b4dbc13a2f45460650..b85f562500ae0c538b23fc99215282f5fda8cff6 100644
--- a/contrib/terraform/openstack/modules/compute/variables.tf
+++ b/contrib/terraform/openstack/modules/compute/variables.tf
@@ -76,6 +76,10 @@ variable "k8s_node_fips" {
type = "list"
}
+variable "k8s_nodes_fips" {
+ type = "map"
+}
+
variable "bastion_fips" {
type = "list"
}
@@ -96,6 +100,8 @@ variable "k8s_allowed_egress_ips" {
type = "list"
}
+variable "k8s_nodes" {}
+
variable "wait_for_floatingip" {}
variable "supplementary_master_groups" {
diff --git a/contrib/terraform/openstack/modules/ips/main.tf b/contrib/terraform/openstack/modules/ips/main.tf
index a4d5cd63726ce40740e04802cc65fed7f26c759a..15f6f1f050bb7a66a8ba416c11ba2c55f7d7f733 100644
--- a/contrib/terraform/openstack/modules/ips/main.tf
+++ b/contrib/terraform/openstack/modules/ips/main.tf
@@ -27,3 +27,10 @@ resource "openstack_networking_floatingip_v2" "bastion" {
pool = "${var.floatingip_pool}"
depends_on = ["null_resource.dummy_dependency"]
}
+
+resource "openstack_networking_floatingip_v2" "k8s_nodes" {
+ for_each = var.number_of_k8s_nodes == 0 ? { for key, value in var.k8s_nodes : key => value if value.floating_ip } : {}
+ pool = "${var.floatingip_pool}"
+ depends_on = ["null_resource.dummy_dependency"]
+}
+
diff --git a/contrib/terraform/openstack/modules/ips/outputs.tf b/contrib/terraform/openstack/modules/ips/outputs.tf
index 703e6f4cd6f6d13f1a64448778244c9c8059f2d9..f74e1c93016800eb091728a98915f3e38f16faed 100644
--- a/contrib/terraform/openstack/modules/ips/outputs.tf
+++ b/contrib/terraform/openstack/modules/ips/outputs.tf
@@ -10,6 +10,10 @@ output "k8s_node_fips" {
value = "${openstack_networking_floatingip_v2.k8s_node[*].address}"
}
+output "k8s_nodes_fips" {
+ value = "${openstack_networking_floatingip_v2.k8s_nodes}"
+}
+
output "bastion_fips" {
value = "${openstack_networking_floatingip_v2.bastion[*].address}"
}
diff --git a/contrib/terraform/openstack/modules/ips/variables.tf b/contrib/terraform/openstack/modules/ips/variables.tf
index a2cb545380e597d390ee670461620501655a2c9b..40e4a759faa844dd79de1079f1ecb7632f8fcf97 100644
--- a/contrib/terraform/openstack/modules/ips/variables.tf
+++ b/contrib/terraform/openstack/modules/ips/variables.tf
@@ -14,4 +14,6 @@ variable "network_name" {}
variable "router_id" {
default = ""
-}
\ No newline at end of file
+}
+
+variable "k8s_nodes" {}
diff --git a/contrib/terraform/openstack/variables.tf b/contrib/terraform/openstack/variables.tf
index b0fe4a6edb99bbabfc838b9f0129ee6c0f6f5253..334e6c678cb1148b6ee60e72f51160d9680a51aa 100644
--- a/contrib/terraform/openstack/variables.tf
+++ b/contrib/terraform/openstack/variables.tf
@@ -225,3 +225,8 @@ variable "router_id" {
description = "uuid of an externally defined router to use"
default = null
}
+
+variable "k8s_nodes" {
+ default = {}
+}
+