From e87d4e9ce3ca15c48534679067dfebcfead2c36a Mon Sep 17 00:00:00 2001
From: Fredrik Liv <fredrik.liv@elastisys.com>
Date: Thu, 7 Oct 2021 19:11:46 +0200
Subject: [PATCH] Added terraform script for Hetzner cloud (#8053)
---
contrib/terraform/hetzner/README.md | 107 +++++++++++++++
contrib/terraform/hetzner/default.tfvars | 44 +++++++
contrib/terraform/hetzner/main.tf | 51 ++++++++
.../modules/kubernetes-cluster/main.tf | 122 ++++++++++++++++++
.../modules/kubernetes-cluster/output.tf | 23 ++++
.../templates/cloud-init.tmpl | 17 +++
.../modules/kubernetes-cluster/variables.tf | 41 ++++++
.../modules/kubernetes-cluster/versions.tf | 9 ++
contrib/terraform/hetzner/output.tf | 7 +
.../terraform/hetzner/templates/inventory.tpl | 16 +++
contrib/terraform/hetzner/variables.tf | 46 +++++++
contrib/terraform/hetzner/versions.tf | 15 +++
12 files changed, 498 insertions(+)
create mode 100644 contrib/terraform/hetzner/README.md
create mode 100644 contrib/terraform/hetzner/default.tfvars
create mode 100644 contrib/terraform/hetzner/main.tf
create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf
create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf
create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl
create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf
create mode 100644 contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf
create mode 100644 contrib/terraform/hetzner/output.tf
create mode 100644 contrib/terraform/hetzner/templates/inventory.tpl
create mode 100644 contrib/terraform/hetzner/variables.tf
create mode 100644 contrib/terraform/hetzner/versions.tf
diff --git a/contrib/terraform/hetzner/README.md b/contrib/terraform/hetzner/README.md
new file mode 100644
index 000000000..747928b33
--- /dev/null
+++ b/contrib/terraform/hetzner/README.md
@@ -0,0 +1,107 @@
+# Kubernetes on Hetzner with Terraform
+
+Provision a Kubernetes cluster on [Hetzner](https://www.hetzner.com/cloud) using Terraform and Kubespray
+
+## Overview
+
+The setup looks like following
+
+```text
+ Kubernetes cluster
++--------------------------+
+| +--------------+ |
+| | +--------------+ |
+| --> | | | |
+| | | Master/etcd | |
+| | | node(s) | |
+| +-+ | |
+| +--------------+ |
+| ^ |
+| | |
+| v |
+| +--------------+ |
+| | +--------------+ |
+| --> | | | |
+| | | Worker | |
+| | | node(s) | |
+| +-+ | |
+| +--------------+ |
++--------------------------+
+```
+
+The nodes uses a private network for node to node communication and a public interface for all external communication.
+
+## Requirements
+
+* Terraform 0.14.0 or newer
+
+## Quickstart
+
+NOTE: Assumes you are at the root of the kubespray repo.
+
+For authentication in your cluster you can use the environment variables.
+
+```bash
+export HCLOUD_TOKEN=api-token
+```
+
+Copy the cluster configuration file.
+
+```bash
+CLUSTER=my-hetzner-cluster
+cp -r inventory/sample inventory/$CLUSTER
+cp contrib/terraform/hetzner/default.tfvars inventory/$CLUSTER/
+cd inventory/$CLUSTER
+```
+
+Edit `default.tfvars` to match your requirement.
+
+Run Terraform to create the infrastructure.
+
+```bash
+terraform init ../../contrib/terraform/hetzner
+terraform apply --var-file default.tfvars ../../contrib/terraform/hetzner/
+```
+
+You should now have a inventory file named `inventory.ini` that you can use with kubespray.
+You can use the inventory file with kubespray to set up a cluster.
+
+It is a good idea to check that you have basic SSH connectivity to the nodes. You can do that by:
+
+```bash
+ansible -i inventory.ini -m ping all
+```
+
+You can setup Kubernetes with kubespray using the generated inventory:
+
+```bash
+ansible-playbook -i inventory.ini ../../cluster.yml -b -v
+```
+
+## Cloud controller
+
+For better support with the cloud you can install the [hcloud cloud controller](https://github.com/hetznercloud/hcloud-cloud-controller-manager) and [CSI driver](https://github.com/hetznercloud/csi-driver).
+
+Please read the instructions in both repos on how to install it.
+
+## Teardown
+
+You can teardown your infrastructure using the following Terraform command:
+
+```bash
+terraform destroy --var-file default.tfvars ../../contrib/terraform/hetzner
+```
+
+## Variables
+
+* `prefix`: Prefix to add to all resources, if set to "" don't set any prefix
+* `ssh_public_keys`: List of public SSH keys to install on all machines
+* `zone`: The zone where to run the cluster
+* `machines`: Machines to provision. Key of this object will be used as the name of the machine
+ * `node_type`: The role of this node *(master|worker)*
+ * `size`: Size of the VM
+ * `image`: The image to use for the VM
+* `ssh_whitelist`: List of IP ranges (CIDR) that will be allowed to ssh to the nodes
+* `api_server_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the API server
+* `nodeport_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to the kubernetes nodes on port 30000-32767 (kubernetes nodeports)
+* `ingress_whitelist`: List of IP ranges (CIDR) that will be allowed to connect to kubernetes workers on port 80 and 443
diff --git a/contrib/terraform/hetzner/default.tfvars b/contrib/terraform/hetzner/default.tfvars
new file mode 100644
index 000000000..cb02b142c
--- /dev/null
+++ b/contrib/terraform/hetzner/default.tfvars
@@ -0,0 +1,44 @@
+prefix = "default"
+zone = "hel1"
+
+inventory_file = "inventory.ini"
+
+ssh_public_keys = [
+ # Put your public SSH key here
+ "ssh-rsa I-did-not-read-the-docs",
+ "ssh-rsa I-did-not-read-the-docs 2",
+]
+
+machines = {
+ "master-0" : {
+ "node_type" : "master",
+ "size" : "cx21",
+ "image" : "ubuntu-20.04",
+ },
+ "worker-0" : {
+ "node_type" : "worker",
+ "size" : "cx21",
+ "image" : "ubuntu-20.04",
+ },
+ "worker-1" : {
+ "node_type" : "worker",
+ "size" : "cx21",
+ "image" : "ubuntu-20.04",
+ }
+}
+
+nodeport_whitelist = [
+ "0.0.0.0/0"
+]
+
+ingress_whitelist = [
+ "0.0.0.0/0"
+]
+
+ssh_whitelist = [
+ "0.0.0.0/0"
+]
+
+api_server_whitelist = [
+ "0.0.0.0/0"
+]
diff --git a/contrib/terraform/hetzner/main.tf b/contrib/terraform/hetzner/main.tf
new file mode 100644
index 000000000..130e89583
--- /dev/null
+++ b/contrib/terraform/hetzner/main.tf
@@ -0,0 +1,51 @@
+provider "hcloud" {}
+
+module "kubernetes" {
+ source = "./modules/kubernetes-cluster"
+
+ prefix = var.prefix
+
+ zone = var.zone
+
+ machines = var.machines
+
+ ssh_public_keys = var.ssh_public_keys
+
+ ssh_whitelist = var.ssh_whitelist
+ api_server_whitelist = var.api_server_whitelist
+ nodeport_whitelist = var.nodeport_whitelist
+ ingress_whitelist = var.ingress_whitelist
+}
+
+#
+# Generate ansible inventory
+#
+
+data "template_file" "inventory" {
+ template = file("${path.module}/templates/inventory.tpl")
+
+ vars = {
+ connection_strings_master = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s etcd_member_name=etcd%d",
+ keys(module.kubernetes.master_ip_addresses),
+ values(module.kubernetes.master_ip_addresses).*.public_ip,
+ values(module.kubernetes.master_ip_addresses).*.private_ip,
+ range(1, length(module.kubernetes.master_ip_addresses) + 1)))
+ connection_strings_worker = join("\n", formatlist("%s ansible_user=ubuntu ansible_host=%s ip=%s",
+ keys(module.kubernetes.worker_ip_addresses),
+ values(module.kubernetes.worker_ip_addresses).*.public_ip,
+ values(module.kubernetes.worker_ip_addresses).*.private_ip))
+
+ list_master = join("\n", keys(module.kubernetes.master_ip_addresses))
+ list_worker = join("\n", keys(module.kubernetes.worker_ip_addresses))
+ }
+}
+
+resource "null_resource" "inventories" {
+ provisioner "local-exec" {
+ command = "echo '${data.template_file.inventory.rendered}' > ${var.inventory_file}"
+ }
+
+ triggers = {
+ template = data.template_file.inventory.rendered
+ }
+}
diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf
new file mode 100644
index 000000000..e8db4e212
--- /dev/null
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/main.tf
@@ -0,0 +1,122 @@
+resource "hcloud_network" "kubernetes" {
+ name = "${var.prefix}-network"
+ ip_range = var.private_network_cidr
+}
+
+resource "hcloud_network_subnet" "kubernetes" {
+ type = "cloud"
+ network_id = hcloud_network.kubernetes.id
+ network_zone = "eu-central"
+ ip_range = var.private_subnet_cidr
+}
+
+resource "hcloud_server" "master" {
+ for_each = {
+ for name, machine in var.machines :
+ name => machine
+ if machine.node_type == "master"
+ }
+
+ name = "${var.prefix}-${each.key}"
+ image = each.value.image
+ server_type = each.value.size
+ location = var.zone
+
+ user_data = templatefile(
+ "${path.module}/templates/cloud-init.tmpl",
+ {
+ ssh_public_keys = var.ssh_public_keys
+ }
+ )
+
+ firewall_ids = [hcloud_firewall.master.id]
+}
+
+resource "hcloud_server_network" "master" {
+ for_each = hcloud_server.master
+
+ server_id = each.value.id
+
+ subnet_id = hcloud_network_subnet.kubernetes.id
+}
+
+resource "hcloud_server" "worker" {
+ for_each = {
+ for name, machine in var.machines :
+ name => machine
+ if machine.node_type == "worker"
+ }
+
+ name = "${var.prefix}-${each.key}"
+ image = each.value.image
+ server_type = each.value.size
+ location = var.zone
+
+ user_data = templatefile(
+ "${path.module}/templates/cloud-init.tmpl",
+ {
+ ssh_public_keys = var.ssh_public_keys
+ }
+ )
+
+ firewall_ids = [hcloud_firewall.worker.id]
+
+}
+
+resource "hcloud_server_network" "worker" {
+ for_each = hcloud_server.worker
+
+ server_id = each.value.id
+
+ subnet_id = hcloud_network_subnet.kubernetes.id
+}
+
+resource "hcloud_firewall" "master" {
+ name = "${var.prefix}-master-firewall"
+
+ rule {
+ direction = "in"
+ protocol = "tcp"
+ port = "22"
+ source_ips = var.ssh_whitelist
+ }
+
+ rule {
+ direction = "in"
+ protocol = "tcp"
+ port = "6443"
+ source_ips = var.api_server_whitelist
+ }
+}
+
+resource "hcloud_firewall" "worker" {
+ name = "${var.prefix}-worker-firewall"
+
+ rule {
+ direction = "in"
+ protocol = "tcp"
+ port = "22"
+ source_ips = var.ssh_whitelist
+ }
+
+ rule {
+ direction = "in"
+ protocol = "tcp"
+ port = "80"
+ source_ips = var.ingress_whitelist
+ }
+
+ rule {
+ direction = "in"
+ protocol = "tcp"
+ port = "443"
+ source_ips = var.ingress_whitelist
+ }
+
+ rule {
+ direction = "in"
+ protocol = "tcp"
+ port = "30000-32767"
+ source_ips = var.nodeport_whitelist
+ }
+}
diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf
new file mode 100644
index 000000000..093647f07
--- /dev/null
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/output.tf
@@ -0,0 +1,23 @@
+output "master_ip_addresses" {
+ value = {
+ for key, instance in hcloud_server.master :
+ instance.name => {
+ "private_ip" = hcloud_server_network.master[key].ip
+ "public_ip" = hcloud_server.master[key].ipv4_address
+ }
+ }
+}
+
+output "worker_ip_addresses" {
+ value = {
+ for key, instance in hcloud_server.worker :
+ instance.name => {
+ "private_ip" = hcloud_server_network.worker[key].ip
+ "public_ip" = hcloud_server.worker[key].ipv4_address
+ }
+ }
+}
+
+output "cluster_private_network_cidr" {
+ value = var.private_subnet_cidr
+}
diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl b/contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl
new file mode 100644
index 000000000..c81aef5dd
--- /dev/null
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/templates/cloud-init.tmpl
@@ -0,0 +1,17 @@
+#cloud-config
+
+users:
+ - default
+ - name: ubuntu
+ shell: /bin/bash
+ sudo: "ALL=(ALL) NOPASSWD:ALL"
+ ssh_authorized_keys:
+ %{ for ssh_public_key in ssh_public_keys ~}
+ - ${ssh_public_key}
+ %{ endfor ~}
+
+ssh_authorized_keys:
+%{ for ssh_public_key in ssh_public_keys ~}
+ - ${ssh_public_key}
+%{ endfor ~}
+
diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf
new file mode 100644
index 000000000..2789ae17b
--- /dev/null
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/variables.tf
@@ -0,0 +1,41 @@
+variable "zone" {
+ type = string
+}
+
+variable "prefix" {}
+
+variable "machines" {
+ type = map(object({
+ node_type = string
+ size = string
+ image = string
+ }))
+}
+
+variable "ssh_public_keys" {
+ type = list(string)
+}
+
+variable "ssh_whitelist" {
+ type = list(string)
+}
+
+variable "api_server_whitelist" {
+ type = list(string)
+}
+
+variable "nodeport_whitelist" {
+ type = list(string)
+}
+
+variable "ingress_whitelist" {
+ type = list(string)
+}
+
+variable "private_network_cidr" {
+ default = "10.0.0.0/16"
+}
+
+variable "private_subnet_cidr" {
+ default = "10.0.10.0/24"
+}
diff --git a/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf b/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf
new file mode 100644
index 000000000..2cea1c20c
--- /dev/null
+++ b/contrib/terraform/hetzner/modules/kubernetes-cluster/versions.tf
@@ -0,0 +1,9 @@
+terraform {
+ required_providers {
+ hcloud = {
+ source = "hetznercloud/hcloud"
+ version = "1.31.1"
+ }
+ }
+ required_version = ">= 0.14"
+}
diff --git a/contrib/terraform/hetzner/output.tf b/contrib/terraform/hetzner/output.tf
new file mode 100644
index 000000000..0336f72ca
--- /dev/null
+++ b/contrib/terraform/hetzner/output.tf
@@ -0,0 +1,7 @@
+output "master_ips" {
+ value = module.kubernetes.master_ip_addresses
+}
+
+output "worker_ips" {
+ value = module.kubernetes.worker_ip_addresses
+}
diff --git a/contrib/terraform/hetzner/templates/inventory.tpl b/contrib/terraform/hetzner/templates/inventory.tpl
new file mode 100644
index 000000000..9c562f4df
--- /dev/null
+++ b/contrib/terraform/hetzner/templates/inventory.tpl
@@ -0,0 +1,16 @@
+[all]
+${connection_strings_master}
+${connection_strings_worker}
+
+[kube-master]
+${list_master}
+
+[etcd]
+${list_master}
+
+[kube-node]
+${list_worker}
+
+[k8s-cluster:children]
+kube-master
+kube-node
diff --git a/contrib/terraform/hetzner/variables.tf b/contrib/terraform/hetzner/variables.tf
new file mode 100644
index 000000000..978575078
--- /dev/null
+++ b/contrib/terraform/hetzner/variables.tf
@@ -0,0 +1,46 @@
+variable "zone" {
+ description = "The zone where to run the cluster"
+}
+
+variable "prefix" {
+ description = "Prefix for resource names"
+ default = "default"
+}
+
+variable "machines" {
+ description = "Cluster machines"
+ type = map(object({
+ node_type = string
+ size = string
+ image = string
+ }))
+}
+
+variable "ssh_public_keys" {
+ description = "Public SSH key which are injected into the VMs."
+ type = list(string)
+}
+
+variable "ssh_whitelist" {
+ description = "List of IP ranges (CIDR) to whitelist for ssh"
+ type = list(string)
+}
+
+variable "api_server_whitelist" {
+ description = "List of IP ranges (CIDR) to whitelist for kubernetes api server"
+ type = list(string)
+}
+
+variable "nodeport_whitelist" {
+ description = "List of IP ranges (CIDR) to whitelist for kubernetes nodeports"
+ type = list(string)
+}
+
+variable "ingress_whitelist" {
+ description = "List of IP ranges (CIDR) to whitelist for HTTP"
+ type = list(string)
+}
+
+variable "inventory_file" {
+ description = "Where to store the generated inventory file"
+}
diff --git a/contrib/terraform/hetzner/versions.tf b/contrib/terraform/hetzner/versions.tf
new file mode 100644
index 000000000..02e5b74ee
--- /dev/null
+++ b/contrib/terraform/hetzner/versions.tf
@@ -0,0 +1,15 @@
+terraform {
+ required_providers {
+ hcloud = {
+ source = "hetznercloud/hcloud"
+ version = "1.31.1"
+ }
+ null = {
+ source = "hashicorp/null"
+ }
+ template = {
+ source = "hashicorp/template"
+ }
+ }
+ required_version = ">= 0.14"
+}
--
GitLab