From 9febd3a37b511ad89d7335e03a0044994e715c00 Mon Sep 17 00:00:00 2001 From: Dominik Wombacher Date: Tue, 5 Nov 2024 09:10:19 +0000 Subject: [PATCH 1/2] feat(rancher): AWS - Variable to adjust security group ingress cidr Default for backwards compatibility 0.0.0.0/0, but recommended to lock down public access to a single ip address or a limited range. Additional change in K3s installation: Advertise kube apiserver on internal ip (advertise-address), allow access in addition through public ip (tls-san). --- rancher/aws/infra.tf | 23 ++++++++++++++++------- rancher/aws/terraform.tfvars.example | 5 ++++- rancher/aws/variables.tf | 6 ++++++ rancher/aws/windows.tf | 2 +- rancher/rancher-common/k3s.tf | 3 ++- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/rancher/aws/infra.tf b/rancher/aws/infra.tf index a1ded029..8da9ed39 100644 --- a/rancher/aws/infra.tf +++ b/rancher/aws/infra.tf @@ -67,17 +67,21 @@ resource "aws_route_table_association" "rancher_route_table_association" { route_table_id = aws_route_table.rancher_route_table.id } -# Security group to allow all traffic -resource "aws_security_group" "rancher_sg_allowall" { - name = "${var.prefix}-rancher-allowall" - description = "Rancher quickstart - allow all traffic" +# Security group to allow ingress and egress traffic +resource "aws_security_group" "rancher_security_group" { + # Adds unique suffix to the SG name, required by lifecycle policy + name_prefix = "${var.prefix}-rancher-security-group" + description = "Rancher quickstart - allow traffic from ${var.security_group_ingress_cidr}" vpc_id = aws_vpc.rancher_vpc.id ingress { from_port = "0" to_port = "0" protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] + cidr_blocks = [ + var.security_group_ingress_cidr + ] + self = true } egress { @@ -90,6 +94,11 @@ resource "aws_security_group" "rancher_sg_allowall" { tags = { Creator = "rancher-quickstart" } + + # Allows changes on existing SG without dependency violation + lifecycle { + create_before_destroy = true + } } # AWS EC2 instance for creating a single node RKE cluster and installing the Rancher server @@ -101,7 +110,7 @@ resource "aws_instance" "rancher_server" { instance_type = var.instance_type key_name = aws_key_pair.quickstart_key_pair.key_name - vpc_security_group_ids = [aws_security_group.rancher_sg_allowall.id] + vpc_security_group_ids = [aws_security_group.rancher_security_group.id] subnet_id = aws_subnet.rancher_subnet.id associate_public_ip_address = true @@ -161,7 +170,7 @@ resource "aws_instance" "quickstart_node" { instance_type = var.instance_type key_name = aws_key_pair.quickstart_key_pair.key_name - vpc_security_group_ids = [aws_security_group.rancher_sg_allowall.id] + vpc_security_group_ids = [aws_security_group.rancher_security_group.id] subnet_id = aws_subnet.rancher_subnet.id associate_public_ip_address = true diff --git a/rancher/aws/terraform.tfvars.example b/rancher/aws/terraform.tfvars.example index c2378bf3..beafcac9 100644 --- a/rancher/aws/terraform.tfvars.example +++ b/rancher/aws/terraform.tfvars.example @@ -42,4 +42,7 @@ rancher_version = "2.7.9" windows_instance_type = "t3a.large" # Kubernetes version to use for managed workload cluster -workload_kubernetes_version = "v1.24.14+rke2r1" \ No newline at end of file +workload_kubernetes_version = "v1.24.14+rke2r1" + +# CIDR that is allowed to access the Rancher server and workload cluster, default: 0.0.0.0/0 +security_group_ingress_cidr = "0.0.0.0/0" \ No newline at end of file diff --git a/rancher/aws/variables.tf b/rancher/aws/variables.tf index a5e1b4a7..c4a68243 100644 --- a/rancher/aws/variables.tf +++ b/rancher/aws/variables.tf @@ -93,6 +93,12 @@ variable "add_windows_node" { default = false } +variable "security_group_ingress_cidr" { + type = string + description = "CIDR that is allowed to access the Rancher server and workload cluster, default: 0.0.0.0/0" + default = "0.0.0.0/0" +} + # Local variables used to reduce repetition locals { node_username = "ec2-user" diff --git a/rancher/aws/windows.tf b/rancher/aws/windows.tf index 20d92a48..bf47fb75 100644 --- a/rancher/aws/windows.tf +++ b/rancher/aws/windows.tf @@ -7,7 +7,7 @@ resource "aws_instance" "quickstart_node_win" { instance_type = var.windows_instance_type key_name = aws_key_pair.quickstart_key_pair.key_name - vpc_security_group_ids = [aws_security_group.rancher_sg_allowall.id] + vpc_security_group_ids = [aws_security_group.rancher_security_group.id] subnet_id = aws_subnet.rancher_subnet.id associate_public_ip_address = true get_password_data = true diff --git a/rancher/rancher-common/k3s.tf b/rancher/rancher-common/k3s.tf index 5ec28f54..1c30ffe6 100644 --- a/rancher/rancher-common/k3s.tf +++ b/rancher/rancher-common/k3s.tf @@ -2,8 +2,9 @@ resource "ssh_resource" "install_k3s" { host = var.node_public_ip + # Advertise kube apiserver on internal ip (advertise-address), allow access in addition through public ip (tls-san) commands = [ - "bash -c 'curl https://get.k3s.io | INSTALL_K3S_EXEC=\"server --node-external-ip ${var.node_public_ip} --node-ip ${var.node_internal_ip}\" INSTALL_K3S_VERSION=${var.rancher_kubernetes_version} sh -'" + "bash -c 'curl https://get.k3s.io | INSTALL_K3S_EXEC=\"server --node-external-ip ${var.node_public_ip} --node-ip ${var.node_internal_ip} --advertise-address ${var.node_internal_ip} --tls-san ${var.node_public_ip}\" INSTALL_K3S_VERSION=${var.rancher_kubernetes_version} sh -'" ] user = var.node_username private_key = var.ssh_private_key_pem From 289682fa7cb0f1e7c2d8f47ec3eb122d120487b3 Mon Sep 17 00:00:00 2001 From: Dominik Wombacher Date: Tue, 5 Nov 2024 11:50:17 +0000 Subject: [PATCH 2/2] feat(rancher): AWS - Split-horizon DNS setup to make rancher reachable on private ip inside vpc Rancher URL is set to a public reachable FQDN based on sslip.io. The workload cluster tries to reach and join Rancher through that public IP. In cases were the ingress CIDR is not 0.0.0.0/0 (recommended), downstream cluster need a way to reach Rancher through the private IP. This is achieved with a split-horizon DNS setup that maps the Rancher FQDN to a private ip inside the VPC. --- rancher/aws/infra.tf | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/rancher/aws/infra.tf b/rancher/aws/infra.tf index 8da9ed39..d47b9e0e 100644 --- a/rancher/aws/infra.tf +++ b/rancher/aws/infra.tf @@ -139,6 +139,28 @@ resource "aws_instance" "rancher_server" { } } +# Split-horizon DNS setup to make rancher reachable through private ip inside vpc +resource "aws_route53_zone" "rancher_route53_private" { + name = "sslip.io" + comment = "${var.prefix}-rancher-route53" + + vpc { + vpc_id = aws_vpc.rancher_vpc.id + } + + tags = { + Name = "${var.prefix}-rancher-route53" + Creator = "rancher-quickstart" + } +} +resource "aws_route53_record" "rancher_sslip_private" { + zone_id = aws_route53_zone.rancher_route53_private.zone_id + name = join(".", ["rancher", aws_instance.rancher_server.public_ip, "sslip.io"]) + type = "A" + ttl = 300 + records = [aws_instance.rancher_server.private_ip] +} + # Rancher resources module "rancher_common" { source = "../rancher-common" @@ -164,7 +186,8 @@ module "rancher_common" { # AWS EC2 instance for creating a single node workload cluster resource "aws_instance" "quickstart_node" { depends_on = [ - aws_route_table_association.rancher_route_table_association + aws_route_table_association.rancher_route_table_association, + aws_route53_record.rancher_sslip_private ] ami = data.aws_ami.sles.id instance_type = var.instance_type