initial commit

.gitignore

@@ -27,3 +27,6 @@ override.tf.json
 
 # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
 # example: *tfplan*
+
+# ides
+.idea

README.md

@@ -0,0 +1,5 @@
+# rke2-aws-tf
+
+__WARNING__: This repository is still a wip.
+
+Terraform IAC for creating HA rke2 clusters on AWS.

common/rke2.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+export INSTALL_RKE2_TYPE="${type}"
+
+if [ "$${DEBUG}" == 1 ]; then
+  set -x
+fi
+
+# info logs the given argument at info log level.
+info() {
+    echo "[INFO] " "$@"
+}
+
+# warn logs the given argument at warn log level.
+warn() {
+    echo "[WARN] " "$@" >&2
+}
+
+# fatal logs the given argument at fatal log level.
+fatal() {
+    echo "[ERROR] " "$@" >&2
+    exit 1
+}
+
+download() {
+  # TODO: Install from repo
+  yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm
+
+  curl -fsSL https://raw.githubusercontent.com/rancher/rke2/master/install.sh | sh -s -
+}
+
+start() {
+  systemctl enable "rke2-$${INSTALL_RKE2_TYPE}"
+  systemctl daemon-reload
+  systemctl start "rke2-$${INSTALL_RKE2_TYPE}"
+}
+
+{
+  download
+  start
+}

data.tf

@@ -0,0 +1,46 @@
+data "template_cloudinit_config" "this" {
+  count = var.server_count
+
+  gzip          = true
+  base64_encode = true
+
+  # Main cloud-init config file
+  part {
+    filename     = "cloud-config.yaml"
+    content_type = "text/cloud-config"
+    content = templatefile("${path.module}/files/cloud-config.yaml", {
+      ssh_authorized_keys = var.ssh_authorized_keys
+    })
+  }
+
+  part {
+    filename     = "00_cfg.sh"
+    content_type = "text/x-shellscript"
+    content = templatefile("${path.module}/files/server.sh", {
+      server_index = count.index
+      server       = "https://${module.cp_lb.dns}:9345"
+
+      args = {
+        "write-kubeconfig-mode" = var.write_kubeconfig_mode
+        "token"                 = random_password.token.result
+      }
+
+      list_args = {
+        "tls-san"                     = [module.cp_lb.dns]
+        "node-label"                  = var.node_labels
+        "node-taint"                  = var.node_taints
+        "kube-apiserver-arg"          = var.kube_apiserver_args
+        "kube-scheduler-arg"          = var.kube_scheduler_args
+        "kube-controller-manager-arg" = var.kube_controller_manager_args
+      }
+    })
+  }
+
+  part {
+    filename     = "01_rke2.sh"
+    content_type = "text/x-shellscript"
+    content = templatefile("${path.module}/common/rke2.sh", {
+      type = "server"
+    })
+  }
+}

examples/quickstart/main.tf

@@ -0,0 +1,65 @@
+provider "aws" {
+  region = "us-gov-west-1"
+}
+
+locals {
+  name    = "quickstart"
+  vpc_id  = "vpc-087496fba26c6d6df"
+  subnets = ["subnet-084b8f063e166cd01", "subnet-0fc3993950d081bfb", "subnet-0b998c99f39ccf748"]
+
+  ami = "ami-24206045"
+
+  tags = {
+    "terraform" = "true",
+    "env"       = "quickstart",
+  }
+}
+
+#
+# Server
+#
+module "rke2" {
+  source = "../.."
+
+  name    = local.name
+  vpc_id  = local.vpc_id
+  subnets = local.subnets
+
+  ssh_authorized_keys = [file("~/.ssh/id_rsa.pub")]
+  ami                 = local.ami
+  server_count        = 3
+
+  tags = local.tags
+}
+
+#
+# Generic agent pool
+#
+module "agents" {
+  source  = "../../modules/agent-nodepool"
+  cluster = module.rke2.cluster_name
+  name    = "generic-agent"
+  vpc_id  = local.vpc_id
+  subnets = local.subnets
+
+  ami                 = local.ami
+  ssh_authorized_keys = [file("~/.ssh/id_rsa.pub")]
+
+  server_url             = module.rke2.server_url
+  token                  = module.rke2.token
+  cluster_security_group = module.rke2.shared_cluster_sg
+}
+
+// For demonstration only, lock down ssh access in production
+resource "aws_security_group_rule" "quickstart_ssh" {
+  from_port         = 22
+  to_port           = 22
+  protocol          = "tcp"
+  security_group_id = module.rke2.shared_cluster_sg
+  type              = "ingress"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+output "rke2" {
+  value = module.rke2
+}

files/cloud-config.yaml

@@ -0,0 +1,14 @@
+#cloud-config
+ssh_authorized_keys:
+%{ for _ in ssh_authorized_keys }
+  - ${_}
+%{ endfor }
+
+users:
+  - default
+  - name: etcd
+    homedir: /var/lib/rancher/rke2/server/db
+    system: true
+  - name: rke2
+    homedir: /var/lib/rancher/rke2
+    system: true

files/server.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+set -e
+
+build_config() {
+  mkdir -p "/etc/rancher/rke2"
+  cat <<-EOF > "/etc/rancher/rke2/config.yaml"
+%{~ if server_index == 0 }
+cluster-init: true
+%{~ else }
+server: "${server}"
+%{~ endif }
+
+# args
+%{~ for k, v in args }
+${k}: "${v}"
+%{~ endfor }
+
+# arg sets
+%{~ for k, v in list_args }
+%{~ if length(v) > 0 }
+${k}:
+  %{~ for _ in v }
+  - "${_}"
+  %{~ endfor }
+%{~ endif }
+%{~ endfor }
+EOF
+}
+
+{
+  build_config
+}

main.tf

@@ -0,0 +1,128 @@
+resource "random_password" "token" {
+  length  = 32
+  special = false
+}
+
+#
+# Controlplane Load Balancer
+#
+module "cp_lb" {
+  source  = "./modules/loadbalancer"
+  name    = var.name
+  vpc_id  = var.vpc_id
+  subnets = var.subnets
+  tags    = var.tags
+}
+
+#
+# Server Nodes
+#
+resource "aws_instance" "servers" {
+  count = var.server_count
+
+  ami              = var.ami
+  instance_type    = var.instance_type
+  subnet_id        = var.subnets[0]
+  user_data_base64 = data.template_cloudinit_config.this[count.index].rendered
+
+  vpc_security_group_ids = [aws_security_group.cluster.id, aws_security_group.server.id]
+
+  root_block_device {
+    volume_size = var.block_device_mappings.size
+    volume_type = "gp2"
+    encrypted   = var.block_device_mappings.encrypted
+  }
+
+  tags = merge({
+    "Name" = "${var.name}-server-${count.index}"
+    "Role" = "server"
+  }, var.tags)
+}
+
+//resource "aws_elb_attachment" "server_lb_attachments" {
+//  count = length(aws_instance.servers)
+//
+//  elb = module.cp_lb.id
+//  instance = aws_instance.servers[count.index].id
+//}
+
+resource "aws_lb_target_group_attachment" "server_tg_attachments" {
+  count = length(aws_instance.servers)
+
+  target_group_arn = module.cp_lb.server_tg_arn
+  target_id        = aws_instance.servers[count.index].id
+}
+
+resource "aws_lb_target_group_attachment" "server_supervisor_tg_attachments" {
+  count = length(aws_instance.servers)
+
+  target_group_arn = module.cp_lb.server_supervisor_tg_arn
+  target_id        = aws_instance.servers[count.index].id
+}
+
+#
+# Shared Cluster Security Group
+#
+resource "aws_security_group" "cluster" {
+  name        = "${var.name}-cluster"
+  description = "Shared ${var.name} cluster security group"
+  vpc_id      = var.vpc_id
+
+  tags = merge({
+
+  }, var.tags)
+}
+
+resource "aws_security_group_rule" "cluster_shared" {
+  description       = "Allow all inbound traffic between cluster nodes"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.cluster.id
+  type              = "ingress"
+
+  self = true
+}
+
+resource "aws_security_group_rule" "cluster_egress" {
+  description       = "Allow all outbound traffic"
+  from_port         = 0
+  to_port           = 0
+  protocol          = "-1"
+  security_group_id = aws_security_group.cluster.id
+  type              = "egress"
+  cidr_blocks       = ["0.0.0.0/0"]
+}
+
+#
+# Shared Server Security Group
+#
+resource "aws_security_group" "server" {
+  name        = "${var.name}-server"
+  description = "Shared ${var.name} server security group"
+  vpc_id      = var.vpc_id
+
+  tags = merge({
+
+  }, var.tags)
+}
+
+resource "aws_security_group_rule" "server_cp" {
+  from_port         = 6443
+  to_port           = 6443
+  protocol          = "tcp"
+  security_group_id = aws_security_group.server.id
+  type              = "ingress"
+  //  source_security_group_id = module.cp_lb.sg
+  cidr_blocks = ["0.0.0.0/0"]
+}
+
+resource "aws_security_group_rule" "server_cp_supervisor" {
+  from_port         = 9345
+  to_port           = 9345
+  protocol          = "tcp"
+  security_group_id = aws_security_group.server.id
+  type              = "ingress"
+  //  source_security_group_id = module.cp_lb.sg
+  cidr_blocks = ["0.0.0.0/0"]
+}

modules/agent-nodepool/data.tf

@@ -0,0 +1,37 @@
+data "template_cloudinit_config" "this" {
+  gzip          = true
+  base64_encode = true
+
+  # Main cloud-init config file
+  part {
+    filename     = "cloud-config.yaml"
+    content_type = "text/cloud-config"
+    content = templatefile("${path.module}/files/cloud-config.yaml", {
+      ssh_authorized_keys = var.ssh_authorized_keys
+    })
+  }
+
+  part {
+    filename     = "00_cfg.sh"
+    content_type = "text/x-shellscript"
+    content = templatefile("${path.module}/files/agent.sh", {
+      args = {
+        "token"  = var.token
+        "server" = var.server_url
+      }
+
+      list_args = {
+        "node-label" = var.node_labels
+        "node-taint" = var.node_taints
+      }
+    })
+  }
+
+  part {
+    filename     = "01_rke2.sh"
+    content_type = "text/x-shellscript"
+    content = templatefile("${path.module}/../../common/rke2.sh", {
+      type = "agent"
+    })
+  }
+}