From b9e5b0cb533684f82e8aee85b639f8edaac6a9de Mon Sep 17 00:00:00 2001
From: Ajarmar <37733838+Ajarmar@users.noreply.github.com>
Date: Wed, 11 May 2022 19:15:03 +0200
Subject: [PATCH] UpCloud server plan, firewall, load balancer integration
(#8758)
* [upcloud] add option to use preconfigured cpu/mem plan
* [upcloud] add option to use firewall rules for API server/SSH access
* [upcloud] add option to use managed load balancer
---
contrib/terraform/upcloud/README.md | 13 ++
.../terraform/upcloud/cluster-settings.tfvars | 37 ++++
contrib/terraform/upcloud/main.tf | 8 +
.../modules/kubernetes-cluster/main.tf | 172 +++++++++++++++++-
.../modules/kubernetes-cluster/output.tf | 4 +
.../modules/kubernetes-cluster/variables.tf | 36 ++++
.../modules/kubernetes-cluster/versions.tf | 2 +-
contrib/terraform/upcloud/output.tf | 4 +
.../upcloud/sample-inventory/cluster.tfvars | 37 ++++
contrib/terraform/upcloud/variables.tf | 44 +++++
contrib/terraform/upcloud/versions.tf | 2 +-
11 files changed, 353 insertions(+), 6 deletions(-)
diff --git a/contrib/terraform/upcloud/README.md b/contrib/terraform/upcloud/README.md
index 6481453fc..0962f8354 100644
--- a/contrib/terraform/upcloud/README.md
+++ b/contrib/terraform/upcloud/README.md
@@ -104,9 +104,22 @@ terraform destroy --var-file cluster-settings.tfvars \
* `zone`: The zone where to run the cluster
* `machines`: Machines to provision. Key of this object will be used as the name of the machine
* `node_type`: The role of this node *(master|worker)*
+ * `plan`: Preconfigured cpu/mem plan to use (disables `cpu` and `mem` attributes below)
* `cpu`: number of cpu cores
* `mem`: memory size in MB
* `disk_size`: The size of the storage in GB
* `additional_disks`: Additional disks to attach to the node.
* `size`: The size of the additional disk in GB
* `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
+* `firewall_enabled`: Enable firewall rules
+* `master_allowed_remote_ips`: List of IP ranges that should be allowed to access API of masters
+ * `start_address`: Start of address range to allow
+ * `end_address`: End of address range to allow
+* `k8s_allowed_remote_ips`: List of IP ranges that should be allowed SSH access to all nodes
+ * `start_address`: Start of address range to allow
+ * `end_address`: End of address range to allow
+* `loadbalancer_enabled`: Enable managed load balancer
+* `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)*
+* `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends
+ * `port`: Port to load balance.
+ * `backend_servers`: List of servers that traffic to the port should be forwarded to.
diff --git a/contrib/terraform/upcloud/cluster-settings.tfvars b/contrib/terraform/upcloud/cluster-settings.tfvars
index 217f46acc..b7bdb2302 100644
--- a/contrib/terraform/upcloud/cluster-settings.tfvars
+++ b/contrib/terraform/upcloud/cluster-settings.tfvars
@@ -20,6 +20,8 @@ ssh_public_keys = [
machines = {
"master-0" : {
"node_type" : "master",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -30,6 +32,8 @@ machines = {
},
"worker-0" : {
"node_type" : "worker",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -49,6 +53,8 @@ machines = {
},
"worker-1" : {
"node_type" : "worker",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -68,6 +74,8 @@ machines = {
},
"worker-2" : {
"node_type" : "worker",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -86,3 +94,32 @@ machines = {
}
}
}
+
+firewall_enabled = false
+
+master_allowed_remote_ips = [
+ {
+ "start_address" : "0.0.0.0"
+ "end_address" : "255.255.255.255"
+ }
+]
+
+k8s_allowed_remote_ips = [
+ {
+ "start_address" : "0.0.0.0"
+ "end_address" : "255.255.255.255"
+ }
+]
+
+loadbalancer_enabled = false
+loadbalancer_plan = "development"
+loadbalancers = {
+ # "http" : {
+ # "port" : 80,
+ # "backend_servers" : [
+ # "worker-0",
+ # "worker-1",
+ # "worker-2"
+ # ]
+ # }
+}
diff --git a/contrib/terraform/upcloud/main.tf b/contrib/terraform/upcloud/main.tf
index b38171cc1..1acc260fa 100644
--- a/contrib/terraform/upcloud/main.tf
+++ b/contrib/terraform/upcloud/main.tf
@@ -22,6 +22,14 @@ module "kubernetes" {
machines = var.machines
ssh_public_keys = var.ssh_public_keys
+
+ firewall_enabled = var.firewall_enabled
+ master_allowed_remote_ips = var.master_allowed_remote_ips
+ k8s_allowed_remote_ips = var.k8s_allowed_remote_ips
+
+ loadbalancer_enabled = var.loadbalancer_enabled
+ loadbalancer_plan = var.loadbalancer_plan
+ loadbalancers = var.loadbalancers
}
#
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
index bce7c924a..ed9de9dd6 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf
@@ -10,6 +10,16 @@ locals {
]
])
+ lb_backend_servers = flatten([
+ for lb_name, loadbalancer in var.loadbalancers : [
+ for backend_server in loadbalancer.backend_servers : {
+ port = loadbalancer.port
+ lb_name = lb_name
+ server_name = backend_server
+ }
+ ]
+ ])
+
# If prefix is set, all resources will be prefixed with "${var.prefix}-"
# Else don't prefix with anything
resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
@@ -45,8 +55,9 @@ resource "upcloud_server" "master" {
}
hostname = "${local.resource-prefix}${each.key}"
- cpu = each.value.cpu
- mem = each.value.mem
+ plan = each.value.plan
+ cpu = each.value.plan == null ? each.value.cpu : null
+ mem = each.value.plan == null ? each.value.mem : null
zone = var.zone
template {
@@ -69,6 +80,8 @@ resource "upcloud_server" "master" {
lifecycle {
ignore_changes = [storage_devices]
}
+
+ firewall = var.firewall_enabled
dynamic "storage_devices" {
for_each = {
@@ -99,8 +112,9 @@ resource "upcloud_server" "worker" {
}
hostname = "${local.resource-prefix}${each.key}"
- cpu = each.value.cpu
- mem = each.value.mem
+ plan = each.value.plan
+ cpu = each.value.plan == null ? each.value.cpu : null
+ mem = each.value.plan == null ? each.value.mem : null
zone = var.zone
template {
@@ -124,6 +138,8 @@ resource "upcloud_server" "worker" {
ignore_changes = [storage_devices]
}
+ firewall = var.firewall_enabled
+
dynamic "storage_devices" {
for_each = {
for disk_key_name, disk in upcloud_storage.additional_disks :
@@ -144,3 +160,151 @@ resource "upcloud_server" "worker" {
create_password = false
}
}
+
+resource "upcloud_firewall_rules" "master" {
+ for_each = upcloud_server.master
+ server_id = each.value.id
+
+ dynamic firewall_rule {
+ for_each = var.master_allowed_remote_ips
+
+ content {
+ action = "accept"
+ comment = "Allow master API access from this network"
+ destination_port_end = "6443"
+ destination_port_start = "6443"
+ direction = "in"
+ family = "IPv4"
+ protocol = "tcp"
+ source_address_end = firewall_rule.value.end_address
+ source_address_start = firewall_rule.value.start_address
+ }
+ }
+
+ dynamic firewall_rule {
+ for_each = length(var.master_allowed_remote_ips) > 0 ? [1] : []
+
+ content {
+ action = "drop"
+ comment = "Deny master API access from other networks"
+ destination_port_end = "6443"
+ destination_port_start = "6443"
+ direction = "in"
+ family = "IPv4"
+ protocol = "tcp"
+ source_address_end = "255.255.255.255"
+ source_address_start = "0.0.0.0"
+ }
+ }
+
+ dynamic firewall_rule {
+ for_each = var.k8s_allowed_remote_ips
+
+ content {
+ action = "accept"
+ comment = "Allow SSH from this network"
+ destination_port_end = "22"
+ destination_port_start = "22"
+ direction = "in"
+ family = "IPv4"
+ protocol = "tcp"
+ source_address_end = firewall_rule.value.end_address
+ source_address_start = firewall_rule.value.start_address
+ }
+ }
+
+ dynamic firewall_rule {
+ for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []
+
+ content {
+ action = "drop"
+ comment = "Deny SSH from other networks"
+ destination_port_end = "22"
+ destination_port_start = "22"
+ direction = "in"
+ family = "IPv4"
+ protocol = "tcp"
+ source_address_end = "255.255.255.255"
+ source_address_start = "0.0.0.0"
+ }
+ }
+}
+
+resource "upcloud_firewall_rules" "k8s" {
+ for_each = upcloud_server.worker
+ server_id = each.value.id
+
+ dynamic firewall_rule {
+ for_each = var.k8s_allowed_remote_ips
+
+ content {
+ action = "accept"
+ comment = "Allow SSH from this network"
+ destination_port_end = "22"
+ destination_port_start = "22"
+ direction = "in"
+ family = "IPv4"
+ protocol = "tcp"
+ source_address_end = firewall_rule.value.end_address
+ source_address_start = firewall_rule.value.start_address
+ }
+ }
+
+ dynamic firewall_rule {
+ for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []
+
+ content {
+ action = "drop"
+ comment = "Deny SSH from other networks"
+ destination_port_end = "22"
+ destination_port_start = "22"
+ direction = "in"
+ family = "IPv4"
+ protocol = "tcp"
+ source_address_end = "255.255.255.255"
+ source_address_start = "0.0.0.0"
+ }
+ }
+}
+
+resource "upcloud_loadbalancer" "lb" {
+ count = var.loadbalancer_enabled ? 1 : 0
+ configured_status = "started"
+ name = "${local.resource-prefix}lb"
+ plan = var.loadbalancer_plan
+ zone = var.zone
+ network = upcloud_network.private.id
+}
+
+resource "upcloud_loadbalancer_backend" "lb_backend" {
+ for_each = var.loadbalancer_enabled ? var.loadbalancers : {}
+
+ loadbalancer = upcloud_loadbalancer.lb[0].id
+ name = "lb-backend-${each.key}"
+}
+
+resource "upcloud_loadbalancer_frontend" "lb_frontend" {
+ for_each = var.loadbalancer_enabled ? var.loadbalancers : {}
+
+ loadbalancer = upcloud_loadbalancer.lb[0].id
+ name = "lb-frontend-${each.key}"
+ mode = "tcp"
+ port = each.value.port
+ default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name
+}
+
+resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
+ for_each = {
+ for be_server in local.lb_backend_servers:
+ "${be_server.server_name}-lb-backend-${be_server.lb_name}" => be_server
+ if var.loadbalancer_enabled
+ }
+
+ backend = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
+ name = "${local.resource-prefix}${each.key}"
+ ip = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
+ port = each.value.port
+ weight = 100
+ max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
+ enabled = true
+}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
index 7343a80bb..c1f8c7c9c 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf
@@ -18,3 +18,7 @@ output "worker_ip" {
}
}
}
+
+output "loadbalancer_domain" {
+ value = var.loadbalancer_enabled ? upcloud_loadbalancer.lb[0].dns_name : null
+}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
index 55abc509b..1fc411b27 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf
@@ -16,6 +16,7 @@ variable "machines" {
description = "Cluster machines"
type = map(object({
node_type = string
+ plan = string
cpu = string
mem = string
disk_size = number
@@ -29,3 +30,38 @@ variable "machines" {
variable "ssh_public_keys" {
type = list(string)
}
+
+variable "firewall_enabled" {
+ type = bool
+}
+
+variable "master_allowed_remote_ips" {
+ type = list(object({
+ start_address = string
+ end_address = string
+ }))
+}
+
+variable "k8s_allowed_remote_ips" {
+ type = list(object({
+ start_address = string
+ end_address = string
+ }))
+}
+
+variable "loadbalancer_enabled" {
+ type = bool
+}
+
+variable "loadbalancer_plan" {
+ type = string
+}
+
+variable "loadbalancers" {
+ description = "Load balancers"
+
+ type = map(object({
+ port = number
+ backend_servers = list(string)
+ }))
+}
diff --git a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
index ffe5d32b1..8d87504df 100644
--- a/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
+++ b/contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf
@@ -3,7 +3,7 @@ terraform {
required_providers {
upcloud = {
source = "UpCloudLtd/upcloud"
- version = "~>2.0.0"
+ version = "~>2.4.0"
}
}
required_version = ">= 0.13"
diff --git a/contrib/terraform/upcloud/output.tf b/contrib/terraform/upcloud/output.tf
index f269c755e..006e3b1cd 100644
--- a/contrib/terraform/upcloud/output.tf
+++ b/contrib/terraform/upcloud/output.tf
@@ -6,3 +6,7 @@ output "master_ip" {
output "worker_ip" {
value = module.kubernetes.worker_ip
}
+
+output "loadbalancer_domain" {
+ value = module.kubernetes.loadbalancer_domain
+}
diff --git a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
index 1400ed3e4..787fb702c 100644
--- a/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/upcloud/sample-inventory/cluster.tfvars
@@ -20,6 +20,8 @@ ssh_public_keys = [
machines = {
"master-0" : {
"node_type" : "master",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -30,6 +32,8 @@ machines = {
},
"worker-0" : {
"node_type" : "worker",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -49,6 +53,8 @@ machines = {
},
"worker-1" : {
"node_type" : "worker",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -68,6 +74,8 @@ machines = {
},
"worker-2" : {
"node_type" : "worker",
+ # plan to use instead of custom cpu/mem
+ "plan" : null,
#number of cpu cores
"cpu" : "2",
#memory size in MB
@@ -86,3 +94,32 @@ machines = {
}
}
}
+
+firewall_enabled = false
+
+master_allowed_remote_ips = [
+ {
+ "start_address" : "0.0.0.0"
+ "end_address" : "255.255.255.255"
+ }
+]
+
+k8s_allowed_remote_ips = [
+ {
+ "start_address" : "0.0.0.0"
+ "end_address" : "255.255.255.255"
+ }
+]
+
+loadbalancer_enabled = false
+loadbalancer_plan = "development"
+loadbalancers = {
+ # "http" : {
+ # "port" : 80,
+ # "backend_servers" : [
+ # "worker-0",
+ # "worker-1",
+ # "worker-2"
+ # ]
+ # }
+}
diff --git a/contrib/terraform/upcloud/variables.tf b/contrib/terraform/upcloud/variables.tf
index 95c90f2f1..60d07c1bf 100644
--- a/contrib/terraform/upcloud/variables.tf
+++ b/contrib/terraform/upcloud/variables.tf
@@ -28,6 +28,7 @@ variable "machines" {
type = map(object({
node_type = string
+ plan = string
cpu = string
mem = string
disk_size = number
@@ -54,3 +55,46 @@ variable "UPCLOUD_USERNAME" {
variable "UPCLOUD_PASSWORD" {
description = "Password for UpCloud API user"
}
+
+variable "firewall_enabled" {
+ description = "Enable firewall rules"
+ default = false
+}
+
+variable "master_allowed_remote_ips" {
+ description = "List of IP start/end addresses allowed to access API of masters"
+ type = list(object({
+ start_address = string
+ end_address = string
+ }))
+ default = []
+}
+
+variable "k8s_allowed_remote_ips" {
+ description = "List of IP start/end addresses allowed to SSH to hosts"
+ type = list(object({
+ start_address = string
+ end_address = string
+ }))
+ default = []
+}
+
+variable "loadbalancer_enabled" {
+ description = "Enable load balancer"
+ default = false
+}
+
+variable "loadbalancer_plan" {
+ description = "Load balancer plan (development/production-small)"
+ default = "development"
+}
+
+variable "loadbalancers" {
+ description = "Load balancers"
+
+ type = map(object({
+ port = number
+ backend_servers = list(string)
+ }))
+ default = {}
+}
diff --git a/contrib/terraform/upcloud/versions.tf b/contrib/terraform/upcloud/versions.tf
index c91089879..5250626c3 100644
--- a/contrib/terraform/upcloud/versions.tf
+++ b/contrib/terraform/upcloud/versions.tf
@@ -3,7 +3,7 @@ terraform {
required_providers {
upcloud = {
source = "UpCloudLtd/upcloud"
- version = "~>2.0.0"
+ version = "~>2.4.0"
}
}
required_version = ">= 0.13"
--
GitLab