From bfebcfa2c57c09e6489d1cf7a9be045a7a0424a8 Mon Sep 17 00:00:00 2001
From: Samuel <faust64@gmail.com>
Date: Fri, 23 Jul 2021 11:43:16 +0200
Subject: [PATCH] fix(misc): contrib/terraform/aws (#7818)
* fix(misc): terraform/aws
- handles deployment with a single availability zone
- handles deployment with more than two availability zone
- handles etcd collocation with control-plane nodes (`aws_etcd_num=0`)
- allows to set a bastion instances count (`aws_bastion_num`)
- allows to set bastion/etcd/control-plane/workers rootfs volume size
- removes variables from terraform.tfvars that were not re-used
- adds .terraform.lock.hcl to .gitignore
- changes/updates base image from ubuntu-18.03 to debian-10
tested by a few coworkers of mine, and myself: thanks for the outstanding
work, on both those terraform samples and kubespray playbooks.
I did not test ubuntu deployments, I could still swap from buster to
focal. LMK.
* fix(gitlab-ci)
AFAIU, terraform.tfvars indentation should be fixed for / no diff
returned running `terraform fmt -check -diff`
https://gitlab.com/kargo-ci/kubernetes-sigs-kubespray/-/jobs/1445622114
---
contrib/terraform/aws/.gitignore | 1 +
.../terraform/aws/create-infrastructure.tf | 30 +++++++++++-----
contrib/terraform/aws/output.tf | 2 +-
.../aws/sample-inventory/cluster.tfvars | 10 ++++--
contrib/terraform/aws/templates/inventory.tpl | 5 ++-
contrib/terraform/aws/terraform.tfvars | 34 ++++++++++++-------
.../terraform/aws/terraform.tfvars.example | 23 +++++++------
contrib/terraform/aws/variables.tf | 20 +++++++++--
8 files changed, 84 insertions(+), 41 deletions(-)
diff --git a/contrib/terraform/aws/.gitignore b/contrib/terraform/aws/.gitignore
index 84fcb5821..373687b80 100644
--- a/contrib/terraform/aws/.gitignore
+++ b/contrib/terraform/aws/.gitignore
@@ -1,2 +1,3 @@
*.tfstate*
+.terraform.lock.hcl
.terraform
diff --git a/contrib/terraform/aws/create-infrastructure.tf b/contrib/terraform/aws/create-infrastructure.tf
index 4a12ecb74..f5ac5617b 100644
--- a/contrib/terraform/aws/create-infrastructure.tf
+++ b/contrib/terraform/aws/create-infrastructure.tf
@@ -20,7 +20,7 @@ module "aws-vpc" {
aws_cluster_name = var.aws_cluster_name
aws_vpc_cidr_block = var.aws_vpc_cidr_block
- aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, 2)
+ aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
aws_cidr_subnets_private = var.aws_cidr_subnets_private
aws_cidr_subnets_public = var.aws_cidr_subnets_public
default_tags = var.default_tags
@@ -31,7 +31,7 @@ module "aws-elb" {
aws_cluster_name = var.aws_cluster_name
aws_vpc_id = module.aws-vpc.aws_vpc_id
- aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, 2)
+ aws_avail_zones = slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names))
aws_subnet_ids_public = module.aws-vpc.aws_subnet_ids_public
aws_elb_api_port = var.aws_elb_api_port
k8s_secure_api_port = var.k8s_secure_api_port
@@ -52,9 +52,9 @@ module "aws-iam" {
resource "aws_instance" "bastion-server" {
ami = data.aws_ami.distro.id
instance_type = var.aws_bastion_size
- count = length(var.aws_cidr_subnets_public)
+ count = var.aws_bastion_num
associate_public_ip_address = true
- availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+ availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
subnet_id = element(module.aws-vpc.aws_subnet_ids_public, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
@@ -79,11 +79,15 @@ resource "aws_instance" "k8s-master" {
count = var.aws_kube_master_num
- availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+ availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
+ root_block_device {
+ volume_size = var.aws_kube_master_disk_size
+ }
+
iam_instance_profile = module.aws-iam.kube_control_plane-profile
key_name = var.AWS_SSH_KEY_NAME
@@ -106,11 +110,15 @@ resource "aws_instance" "k8s-etcd" {
count = var.aws_etcd_num
- availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+ availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
+ root_block_device {
+ volume_size = var.aws_etcd_disk_size
+ }
+
key_name = var.AWS_SSH_KEY_NAME
tags = merge(var.default_tags, tomap({
@@ -126,11 +134,15 @@ resource "aws_instance" "k8s-worker" {
count = var.aws_kube_worker_num
- availability_zone = element(slice(data.aws_availability_zones.available.names, 0, 2), count.index)
+ availability_zone = element(slice(data.aws_availability_zones.available.names, 0, length(var.aws_cidr_subnets_public) <= length(data.aws_availability_zones.available.names) ? length(var.aws_cidr_subnets_public) : length(data.aws_availability_zones.available.names)), count.index)
subnet_id = element(module.aws-vpc.aws_subnet_ids_private, count.index)
vpc_security_group_ids = module.aws-vpc.aws_security_group
+ root_block_device {
+ volume_size = var.aws_kube_worker_disk_size
+ }
+
iam_instance_profile = module.aws-iam.kube-worker-profile
key_name = var.AWS_SSH_KEY_NAME
@@ -152,10 +164,10 @@ data "template_file" "inventory" {
public_ip_address_bastion = join("\n", formatlist("bastion ansible_host=%s", aws_instance.bastion-server.*.public_ip))
connection_strings_master = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-master.*.private_dns, aws_instance.k8s-master.*.private_ip))
connection_strings_node = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-worker.*.private_dns, aws_instance.k8s-worker.*.private_ip))
- connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", aws_instance.k8s-etcd.*.private_dns, aws_instance.k8s-etcd.*.private_ip))
list_master = join("\n", aws_instance.k8s-master.*.private_dns)
list_node = join("\n", aws_instance.k8s-worker.*.private_dns)
- list_etcd = join("\n", aws_instance.k8s-etcd.*.private_dns)
+ connection_strings_etcd = join("\n", formatlist("%s ansible_host=%s", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)), ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip))))
+ list_etcd = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_dns) : (aws_instance.k8s-master.*.private_dns)))
elb_api_fqdn = "apiserver_loadbalancer_domain_name=\"${module.aws-elb.aws_elb_api_fqdn}\""
}
}
diff --git a/contrib/terraform/aws/output.tf b/contrib/terraform/aws/output.tf
index 35e6e27a7..8cac230af 100644
--- a/contrib/terraform/aws/output.tf
+++ b/contrib/terraform/aws/output.tf
@@ -11,7 +11,7 @@ output "workers" {
}
output "etcd" {
- value = join("\n", aws_instance.k8s-etcd.*.private_ip)
+ value = join("\n", ((var.aws_etcd_num > 0) ? (aws_instance.k8s-etcd.*.private_ip) : (aws_instance.k8s-master.*.private_ip)))
}
output "aws_elb_api_fqdn" {
diff --git a/contrib/terraform/aws/sample-inventory/cluster.tfvars b/contrib/terraform/aws/sample-inventory/cluster.tfvars
index 1b0b76c97..d731a0416 100644
--- a/contrib/terraform/aws/sample-inventory/cluster.tfvars
+++ b/contrib/terraform/aws/sample-inventory/cluster.tfvars
@@ -9,6 +9,8 @@ aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]
#Bastion Host
+aws_bastion_num = 1
+
aws_bastion_size = "t2.medium"
#Kubernetes Cluster
@@ -17,22 +19,26 @@ aws_kube_master_num = 3
aws_kube_master_size = "t2.medium"
+aws_kube_master_disk_size = 50
+
aws_etcd_num = 3
aws_etcd_size = "t2.medium"
+aws_etcd_disk_size = 50
+
aws_kube_worker_num = 4
aws_kube_worker_size = "t2.medium"
+aws_kube_worker_disk_size = 50
+
#Settings AWS ELB
aws_elb_api_port = 6443
k8s_secure_api_port = 6443
-kube_insecure_apiserver_address = "0.0.0.0"
-
default_tags = {
# Env = "devtest" # Product = "kubernetes"
}
diff --git a/contrib/terraform/aws/templates/inventory.tpl b/contrib/terraform/aws/templates/inventory.tpl
index baa9ea854..c0d0d1024 100644
--- a/contrib/terraform/aws/templates/inventory.tpl
+++ b/contrib/terraform/aws/templates/inventory.tpl
@@ -10,19 +10,18 @@ ${public_ip_address_bastion}
[kube_control_plane]
${list_master}
-
[kube_node]
${list_node}
-
[etcd]
${list_etcd}
+[calico_rr]
[k8s_cluster:children]
kube_node
kube_control_plane
-
+calico_rr
[k8s_cluster:vars]
${elb_api_fqdn}
diff --git a/contrib/terraform/aws/terraform.tfvars b/contrib/terraform/aws/terraform.tfvars
index c8db6b424..21089ebdd 100644
--- a/contrib/terraform/aws/terraform.tfvars
+++ b/contrib/terraform/aws/terraform.tfvars
@@ -6,26 +6,34 @@ aws_vpc_cidr_block = "10.250.192.0/18"
aws_cidr_subnets_private = ["10.250.192.0/20", "10.250.208.0/20"]
aws_cidr_subnets_public = ["10.250.224.0/20", "10.250.240.0/20"]
-#Bastion Host
-aws_bastion_size = "t2.medium"
+# single AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/20"]
+#aws_cidr_subnets_public = ["10.250.224.0/20"]
+# 3+ AZ deployment
+#aws_cidr_subnets_private = ["10.250.192.0/24","10.250.193.0/24","10.250.194.0/24","10.250.195.0/24"]
+#aws_cidr_subnets_public = ["10.250.224.0/24","10.250.225.0/24","10.250.226.0/24","10.250.227.0/24"]
-#Kubernetes Cluster
+#Bastion Host
+aws_bastion_num = 1
+aws_bastion_size = "t3.small"
-aws_kube_master_num = 3
-aws_kube_master_size = "t2.medium"
+#Kubernetes Cluster
+aws_kube_master_num = 3
+aws_kube_master_size = "t3.medium"
+aws_kube_master_disk_size = 50
-aws_etcd_num = 3
-aws_etcd_size = "t2.medium"
+aws_etcd_num = 0
+aws_etcd_size = "t3.medium"
+aws_etcd_disk_size = 50
-aws_kube_worker_num = 4
-aws_kube_worker_size = "t2.medium"
+aws_kube_worker_num = 4
+aws_kube_worker_size = "t3.medium"
+aws_kube_worker_disk_size = 50
#Settings AWS ELB
-
-aws_elb_api_port = 6443
-k8s_secure_api_port = 6443
-kube_insecure_apiserver_address = "0.0.0.0"
+aws_elb_api_port = 6443
+k8s_secure_api_port = 6443
default_tags = {
# Env = "devtest"
diff --git a/contrib/terraform/aws/terraform.tfvars.example b/contrib/terraform/aws/terraform.tfvars.example
index 666b21db2..76684d831 100644
--- a/contrib/terraform/aws/terraform.tfvars.example
+++ b/contrib/terraform/aws/terraform.tfvars.example
@@ -8,25 +8,26 @@ aws_cidr_subnets_public = ["10.250.224.0/20","10.250.240.0/20"]
aws_avail_zones = ["eu-central-1a","eu-central-1b"]
#Bastion Host
-aws_bastion_ami = "ami-5900cc36"
-aws_bastion_size = "t2.small"
-
+aws_bastion_num = 1
+aws_bastion_size = "t3.small"
#Kubernetes Cluster
-
aws_kube_master_num = 3
-aws_kube_master_size = "t2.medium"
+aws_kube_master_size = "t3.medium"
+aws_kube_master_disk_size = 50
aws_etcd_num = 3
-aws_etcd_size = "t2.medium"
+aws_etcd_size = "t3.medium"
+aws_etcd_disk_size = 50
aws_kube_worker_num = 4
-aws_kube_worker_size = "t2.medium"
-
-aws_cluster_ami = "ami-903df7ff"
+aws_kube_worker_size = "t3.medium"
+aws_kube_worker_disk_size = 50
#Settings AWS ELB
-
aws_elb_api_port = 6443
k8s_secure_api_port = 6443
-kube_insecure_apiserver_address = 0.0.0.0
+
+default_tags = { }
+
+inventory_file = "../../../inventory/hosts"
diff --git a/contrib/terraform/aws/variables.tf b/contrib/terraform/aws/variables.tf
index 19d165f04..92a5512c8 100644
--- a/contrib/terraform/aws/variables.tf
+++ b/contrib/terraform/aws/variables.tf
@@ -25,7 +25,7 @@ data "aws_ami" "distro" {
filter {
name = "name"
- values = ["ubuntu/images/hvm-ssd/ubuntu-bionic-18.04-amd64-server-*"]
+ values = ["debian-10-amd64-*"]
}
filter {
@@ -33,7 +33,7 @@ data "aws_ami" "distro" {
values = ["hvm"]
}
- owners = ["099720109477"] # Canonical
+ owners = ["136693071363"] # Debian-10
}
//AWS VPC Variables
@@ -63,10 +63,18 @@ variable "aws_bastion_size" {
* The number should be divisable by the number of used
* AWS Availability Zones without an remainder.
*/
+variable "aws_bastion_num" {
+ description = "Number of Bastion Nodes"
+}
+
variable "aws_kube_master_num" {
description = "Number of Kubernetes Master Nodes"
}
+variable "aws_kube_master_disk_size" {
+ description = "Disk size for Kubernetes Master Nodes (in GiB)"
+}
+
variable "aws_kube_master_size" {
description = "Instance size of Kube Master Nodes"
}
@@ -75,6 +83,10 @@ variable "aws_etcd_num" {
description = "Number of etcd Nodes"
}
+variable "aws_etcd_disk_size" {
+ description = "Disk size for etcd Nodes (in GiB)"
+}
+
variable "aws_etcd_size" {
description = "Instance size of etcd Nodes"
}
@@ -83,6 +95,10 @@ variable "aws_kube_worker_num" {
description = "Number of Kubernetes Worker Nodes"
}
+variable "aws_kube_worker_disk_size" {
+ description = "Disk size for Kubernetes Worker Nodes (in GiB)"
+}
+
variable "aws_kube_worker_size" {
description = "Instance size of Kubernetes Worker Nodes"
}
--
GitLab