From aa00c1d91ac9f4b6053b10aacd3fd1a6b7f50381 Mon Sep 17 00:00:00 2001
From: Fredrik Liv <fredrik.liv@elastisys.com>
Date: Fri, 10 Sep 2021 22:55:21 +0200
Subject: [PATCH] Updated UpCloud terraform script to use private network and
dynamic (#7779)
additional disks
---
contrib/terraform/upcloud/README.md | 48 +++---
.../terraform/upcloud/cluster-settings.tfvars | 40 ++++-
contrib/terraform/upcloud/main.tf | 16 +-
.../modules/kubernetes-cluster/main.tf | 142 +++++++++++++-----
.../modules/kubernetes-cluster/output.tf | 10 +-
.../modules/kubernetes-cluster/variables.tf | 24 +--
.../upcloud/sample-inventory/cluster.tfvars | 40 ++++-
contrib/terraform/upcloud/variables.tf | 33 +++-
8 files changed, 264 insertions(+), 89 deletions(-)
diff --git a/contrib/terraform/upcloud/README.md b/contrib/terraform/upcloud/README.md
index dcce236fe..6481453fc 100644
--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -8,27 +8,29 @@ The setup looks like following
```text
Kubernetes cluster
-+-----------------------+
-| +--------------+ |
-| | +--------------+ |
-| | | | |
-| | | Master/etcd | |
-| | | node(s) | |
-| +-+ | |
-| +--------------+ |
-| ^ |
-| | |
-| v |
-| +--------------+ |
-| | +--------------+ |
-| | | | |
-| | | Worker | |
-| | | node(s) | |
-| +-+ | |
-| +--------------+ |
-+-----------------------+
++--------------------------+
+| +--------------+ |
+| | +--------------+ |
+| --> | | | |
+| | | Master/etcd | |
+| | | node(s) | |
+| +-+ | |
+| +--------------+ |
+| ^ |
+| | |
+| v |
+| +--------------+ |
+| | +--------------+ |
+| --> | | | |
+| | | Worker | |
+| | | node(s) | |
+| +-+ | |
+| +--------------+ |
++--------------------------+
```
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
## Requirements
* Terraform 0.13.0 or newer
@@ -94,9 +96,10 @@ terraform destroy --var-file cluster-settings.tfvars \
## Variables
-* `hostname`: A valid domain name, e.g. example.com. The maximum length is 128 characters.
+* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
* `template_name`: The name or UUID of a base image
-* `username`: a user to access the nodes
+* `username`: a user to access the nodes, defaults to "ubuntu"
+* `private_network_cidr`: CIDR to use for the private network, defaults to "172.16.0.0/24"
* `ssh_public_keys`: List of public SSH keys to install on all machines
* `zone`: The zone where to run the cluster
* `machines`: Machines to provision. Key of this object will be used as the name of the machine
@@ -104,3 +107,6 @@ terraform destroy --var-file cluster-settings.tfvars \
* `cpu`: number of cpu cores
* `mem`: memory size in MB
* `disk_size`: The size of the storage in GB
+ * `additional_disks`: Additional disks to attach to the node.
+ * `size`: The size of the additional disk in GB
+ * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars
index 08bf5dacf..217f46acc 100644
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -1,12 +1,11 @@
-
# See: https://developers.upcloud.com/1.3/5-zones/
zone = "fi-hel1"
username = "ubuntu"
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
-# A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
-hostname = "example.com"
+inventory_file = "inventory.ini"
# Set the operating system using UUID or exact name
template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
@@ -17,7 +16,7 @@ ssh_public_keys = [
"ssh-rsa public key 2",
]
-#check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
machines = {
"master-0" : {
"node_type" : "master",
@@ -27,6 +26,7 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks" : {}
},
"worker-0" : {
"node_type" : "worker",
@@ -36,6 +36,16 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks" : {
+ # "some-disk-name-1": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # },
+ # "some-disk-name-2": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # }
+ }
},
"worker-1" : {
"node_type" : "worker",
@@ -45,6 +55,16 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks" : {
+ # "some-disk-name-1": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # },
+ # "some-disk-name-2": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # }
+ }
},
"worker-2" : {
"node_type" : "worker",
@@ -54,5 +74,15 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks" : {
+ # "some-disk-name-1": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # },
+ # "some-disk-name-2": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # }
+ }
}
}
diff --git a/contrib/terraform/upcloud/main.tf b/contrib/terraform/upcloud/main.tf
index 8ddd46542..b38171cc1 100644
--- a/contrib/terraform/upcloud/main.tf
+++ b/contrib/terraform/upcloud/main.tf
@@ -11,12 +11,14 @@ provider "upcloud" {
module "kubernetes" {
source = "./modules/kubernetes-cluster"
- zone = var.zone
- hostname = var.hostname
+ prefix = var.prefix
+ zone = var.zone
template_name = var.template_name
username = var.username
+ private_network_cidr = var.private_network_cidr
+
machines = var.machines
ssh_public_keys = var.ssh_public_keys
@@ -30,13 +32,15 @@ data "template_file" "inventory" {
template = file("${path.module}/templates/inventory.tpl")
vars = {
- connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s etcd_member_name=etcd%d",
+ connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
keys(module.kubernetes.master_ip),
- values(module.kubernetes.master_ip),
+ values(module.kubernetes.master_ip).*.public_ip,
+ values(module.kubernetes.master_ip).*.private_ip,
range(1, length(module.kubernetes.master_ip) + 1)))
- connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s",
+ connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
keys(module.kubernetes.worker_ip),
- values(module.kubernetes.worker_ip)))
+ values(module.kubernetes.worker_ip).*.public_ip,
+ values(module.kubernetes.worker_ip).*.private_ip))
list_master = join("\n", formatlist("%s",
keys(module.kubernetes.master_ip)))
list_worker = join("\n", formatlist("%s",
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
index 6a79af720..c72788c69 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -1,3 +1,41 @@
+locals {
+ # Create a list of all disks to create
+ disks = flatten([
+ for node_name, machine in var.machines : [
+ for disk_name, disk in machine.additional_disks : {
+ disk = disk
+ disk_name = disk_name
+ node_name = node_name
+ }
+ ]
+ ])
+
+ # If prefix is set, all resources will be prefixed with "${var.prefix}-"
+ # Else don't prefix with anything
+ resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
+}
+
+resource "upcloud_network" "private" {
+ name = "${local.resource-prefix}k8s-network"
+ zone = var.zone
+
+ ip_network {
+ address = var.private_network_cidr
+ dhcp = true
+ family = "IPv4"
+ }
+}
+
+resource "upcloud_storage" "additional_disks" {
+ for_each = {
+ for disk in local.disks: "${disk.node_name}_${disk.disk_name}" => disk.disk
+ }
+
+ size = each.value.size
+ tier = each.value.tier
+ title = "${local.resource-prefix}${each.key}"
+ zone = var.zone
+}
resource "upcloud_server" "master" {
for_each = {
@@ -6,35 +44,48 @@ resource "upcloud_server" "master" {
if machine.node_type == "master"
}
- hostname = "${each.key}.${var.hostname}"
- cpu = each.value.cpu
- mem = each.value.mem
- zone = var.zone
+ hostname = "${local.resource-prefix}${each.key}"
+ cpu = each.value.cpu
+ mem = each.value.mem
+ zone = var.zone
template {
storage = var.template_name
- size = each.value.disk_size
+ size = each.value.disk_size
+ }
+
+ # Public network interface
+ network_interface {
+ type = "public"
}
- # Network interfaces
- network_interface {
- type = "public"
- }
+ # Private network interface
+ network_interface {
+ type = "private"
+ network = upcloud_network.private.id
+ }
- network_interface {
- type = "utility"
- }
- # Include at least one public SSH key
- login {
- user = var.username
- keys = var.ssh_public_keys
- create_password = false
+ dynamic "storage_devices" {
+ for_each = {
+ for disk_key_name, disk in upcloud_storage.additional_disks :
+ disk_key_name => disk
+ # Only add the disk if it matches the node name in the start of its name
+ if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+ }
- }
+ content {
+ storage = storage_devices.value.id
+ }
+ }
+ # Include at least one public SSH key
+ login {
+ user = var.username
+ keys = var.ssh_public_keys
+ create_password = false
+ }
}
-
resource "upcloud_server" "worker" {
for_each = {
for name, machine in var.machines :
@@ -42,25 +93,44 @@ resource "upcloud_server" "worker" {
if machine.node_type == "worker"
}
- hostname = "${each.key}.${var.hostname}"
- cpu = each.value.cpu
- mem = each.value.mem
- zone = var.zone
+ hostname = "${local.resource-prefix}${each.key}"
+ cpu = each.value.cpu
+ mem = each.value.mem
+ zone = var.zone
template {
- storage = var.template_name
- size = each.value.disk_size
+ storage = var.template_name
+ size = each.value.disk_size
+ }
+
+ # Public network interface
+ network_interface {
+ type = "public"
}
- # Network interfaces
- network_interface {
- type = "public"
- }
-
- # Include at least one public SSH key
- login {
- user = var.username
- keys = var.ssh_public_keys
- create_password = false
- }
+ # Private network interface
+ network_interface {
+ type = "private"
+ network = upcloud_network.private.id
+ }
+
+ dynamic "storage_devices" {
+ for_each = {
+ for disk_key_name, disk in upcloud_storage.additional_disks :
+ disk_key_name => disk
+ # Only add the disk if it matches the node name in the start of its name
+ if length(regexall("^${each.key}_.+", disk_key_name)) > 0
+ }
+
+ content {
+ storage = storage_devices.value.id
+ }
+ }
+
+ # Include at least one public SSH key
+ login {
+ user = var.username
+ keys = var.ssh_public_keys
+ create_password = false
+ }
}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
index 2661ac013..7343a80bb 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -2,13 +2,19 @@
output "master_ip" {
value = {
for instance in upcloud_server.master :
- instance.hostname => instance.network_interface[0].ip_address
+ instance.hostname => {
+ "public_ip": instance.network_interface[0].ip_address
+ "private_ip": instance.network_interface[1].ip_address
+ }
}
}
output "worker_ip" {
value = {
for instance in upcloud_server.worker :
- instance.hostname => instance.network_interface[0].ip_address
+ instance.hostname => {
+ "public_ip": instance.network_interface[0].ip_address
+ "private_ip": instance.network_interface[1].ip_address
+ }
}
}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
index 5b130ad10..55abc509b 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -1,22 +1,28 @@
-variable "zone" {
+variable "prefix" {
type = string
}
-variable "hostname"{
- default ="example.com"
+variable "zone" {
+ type = string
}
-variable "template_name"{}
+variable "template_name" {}
+
+variable "username" {}
-variable "username"{}
+variable "private_network_cidr" {}
variable "machines" {
description = "Cluster machines"
type = map(object({
- node_type = string
- cpu = string
- mem = string
- disk_size = number
+ node_type = string
+ cpu = string
+ mem = string
+ disk_size = number
+ additional_disks = map(object({
+ size = number
+ tier = string
+ }))
}))
}
diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
index 0a324045e..1400ed3e4 100644
--- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
@@ -2,20 +2,21 @@
zone = "fi-hel1"
username = "ubuntu"
-inventory_file = "inventory.ini"
+# Prefix to use for all resources to separate them from other resources
+prefix = "kubespray"
-# A valid domain name, e.g. host.example.com. The maximum length is 128 characters.
-hostname = "example.com"
+inventory_file = "inventory.ini"
# Set the operating system using UUID or exact name
template_name = "Ubuntu Server 20.04 LTS (Focal Fossa)"
+
ssh_public_keys = [
# Put your public SSH key here
"ssh-rsa I-did-not-read-the-docs",
"ssh-rsa I-did-not-read-the-docs 2",
]
-check list of available plan https://developers.upcloud.com/1.3/7-plans/
+# check list of available plan https://developers.upcloud.com/1.3/7-plans/
machines = {
"master-0" : {
"node_type" : "master",
@@ -25,6 +26,7 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks": {}
},
"worker-0" : {
"node_type" : "worker",
@@ -34,6 +36,16 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks": {
+ # "some-disk-name-1": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # },
+ # "some-disk-name-2": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # }
+ }
},
"worker-1" : {
"node_type" : "worker",
@@ -43,6 +55,16 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks": {
+ # "some-disk-name-1": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # },
+ # "some-disk-name-2": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # }
+ }
},
"worker-2" : {
"node_type" : "worker",
@@ -52,5 +74,15 @@ machines = {
"mem" : "4096"
# The size of the storage in GB
"disk_size" : 250
+ "additional_disks": {
+ # "some-disk-name-1": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # },
+ # "some-disk-name-2": {
+ # "size": 100,
+ # "tier": "maxiops",
+ # }
+ }
}
}
diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf
index 60941d572..95c90f2f1 100644
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -1,23 +1,40 @@
+variable "prefix" {
+ type = string
+ default = "kubespray"
+
+ description = "Prefix that is used to distinguish these resources from others"
+}
variable "zone" {
description = "The zone where to run the cluster"
}
-variable "hostname" {
- default = "example.com"
+variable "template_name" {
+ description = "Block describing the preconfigured operating system"
}
-variable "template_name" {}
+variable "username" {
+ description = "The username to use for the nodes"
+ default = "ubuntu"
+}
-variable "username" {}
+variable "private_network_cidr" {
+ description = "CIDR to use for the private network"
+ default = "172.16.0.0/24"
+}
variable "machines" {
description = "Cluster machines"
+
type = map(object({
node_type = string
cpu = string
mem = string
disk_size = number
+ additional_disks = map(object({
+ size = number
+ tier = string
+ }))
}))
}
@@ -30,6 +47,10 @@ variable "inventory_file" {
description = "Where to store the generated inventory file"
}
-variable "UPCLOUD_USERNAME" {}
+variable "UPCLOUD_USERNAME" {
+ description = "UpCloud username with API access"
+}
-variable "UPCLOUD_PASSWORD" {}
+variable "UPCLOUD_PASSWORD" {
+ description = "Password for UpCloud API user"
+}
--
GitLab