From 21f6db82c5bb9ac0c3f6bb67f4897480531d98dc Mon Sep 17 00:00:00 2001
From: Xi Bai <baixiac@gmail.com>
Date: Mon, 25 Mar 2024 17:53:32 +0000
Subject: [PATCH] schedule coredns replicas onto different nodes for HA

---
 cluster/eks.tf          | 47 +++++++++++++++++++++++++----------------
 config/karpenter.tf     |  8 +++----
 config/terraform.tfvars | 22 +++++++++----------
 config/variables.tf     |  4 ++--
 4 files changed, 46 insertions(+), 35 deletions(-)

diff --git a/cluster/eks.tf b/cluster/eks.tf
index 089bb20..292f83c 100644
--- a/cluster/eks.tf
+++ b/cluster/eks.tf
@@ -112,34 +112,45 @@ module "eks" {
 
   cluster_addons = {
     coredns = {
-      addon_version     = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.coredns
-      resolve_conflicts = "OVERWRITE"
-      configuration_values = var.create_dmz_node_group ? jsonencode({
-        tolerations : [
+      addon_version               = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.coredns
+      resolve_conflicts_on_create = "OVERWRITE"
+      configuration_values = jsonencode({
+        tolerations : var.create_dmz_node_group ? [
           {
             key : "dmz-pod",
             operator : "Equal",
             value : "yes",
             effect : "NoExecute"
           }
-        ],
-        nodeSelector : {
+        ] : [],
+        nodeSelector : var.create_dmz_node_group ? {
           role : "dmz-1"
+        } : {},
+        affinity : {
+          podAntiAffinity : {
+            requiredDuringSchedulingIgnoredDuringExecution : [{
+              labelSelector : {
+                matchExpressions : [{
+                  key : "k8s-app"
+                  operator : "In"
+                  values : ["kube-dns"]
+                }]
+              },
+              topologyKey : "kubernetes.io/hostname"
+            }]
+          }
         }
-        }) : jsonencode({
-        tolerations : [],
-        nodeSelector : {}
       })
     }
     kube-proxy = {
-      addon_version     = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.kube_proxy
-      resolve_conflicts = "OVERWRITE"
+      addon_version               = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.kube_proxy
+      resolve_conflicts_on_create = "OVERWRITE"
     }
     vpc-cni = {
-      addon_version            = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.vpc_cni
-      resolve_conflicts        = "OVERWRITE"
-      before_compute           = true
-      service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
+      addon_version               = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.vpc_cni
+      resolve_conflicts_on_create = "OVERWRITE"
+      before_compute              = true
+      service_account_role_arn    = module.vpc_cni_irsa.iam_role_arn
       configuration_values = jsonencode({
         env : {
           # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html
@@ -149,9 +160,9 @@ module "eks" {
       })
     }
     aws-ebs-csi-driver = {
-      addon_version            = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.ebs_csi_driver
-      resolve_conflicts        = "OVERWRITE"
-      service_account_role_arn = module.ebs_csi_irsa.iam_role_arn
+      addon_version               = local.eks_core_versions[var.eks_kubernetes_version].cluster_addons.ebs_csi_driver
+      resolve_conflicts_on_create = "OVERWRITE"
+      service_account_role_arn    = module.ebs_csi_irsa.iam_role_arn
       configuration_values = jsonencode({
         sidecars : {
           snapshotter : {
diff --git a/config/karpenter.tf b/config/karpenter.tf
index 8e06c6e..0708926 100644
--- a/config/karpenter.tf
+++ b/config/karpenter.tf
@@ -27,15 +27,15 @@ locals {
     },
     {
       name  = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn"
-      value = module.karpenter[0].irsa_arn
+      value = length(module.karpenter) > 0 ? module.karpenter[0].irsa_arn : null
     },
     {
       name  = "settings.aws.defaultInstanceProfile"
-      value = module.karpenter[0].instance_profile_name
+      value = length(module.karpenter) > 0 ? module.karpenter[0].instance_profile_name : null
     },
     {
       name  = "settings.aws.interruptionQueueName"
-      value = module.karpenter[0].queue_name
+      value = length(module.karpenter) > 0 ? module.karpenter[0].queue_name : null
     },
     {
       name  = "replicas"
@@ -77,7 +77,7 @@ resource "helm_release" "karpenter" {
 
 
   dynamic "set" {
-    for_each = var.create_dmz_node_group ? concat(local.common_settings, local.tolerations_settings) : local.common_settings
+    for_each = var.with_dmz_pods ? concat(local.common_settings, local.tolerations_settings) : local.common_settings
 
     content {
       name  = set.value.name
diff --git a/config/terraform.tfvars b/config/terraform.tfvars
index 18f6af1..71ca455 100644
--- a/config/terraform.tfvars
+++ b/config/terraform.tfvars
@@ -1,11 +1,11 @@
-AWS_REGION            = "eu-west-2"
-environment           = "dev"
-domain_name           = "change-me-radar-base-dummy-domain.net"
-create_dmz_node_group = false
-enable_karpenter      = false
-enable_msk            = false
-enable_rds            = false
-enable_route53        = false
-enable_ses            = false
-enable_s3             = false
-enable_eip            = false
\ No newline at end of file
+AWS_REGION       = "eu-west-2"
+environment      = "dev"
+domain_name      = "change-me-radar-base-dummy-domain.net"
+with_dmz_pods    = false
+enable_karpenter = false
+enable_msk       = false
+enable_rds       = false
+enable_route53   = false
+enable_ses       = false
+enable_s3        = false
+enable_eip       = false
\ No newline at end of file
diff --git a/config/variables.tf b/config/variables.tf
index 065267d..f71dc31 100644
--- a/config/variables.tf
+++ b/config/variables.tf
@@ -89,9 +89,9 @@ variable "radar_postgres_password" {
   sensitive = true
 }
 
-variable "create_dmz_node_group" {
+variable "with_dmz_pods" {
   type        = bool
-  description = "Whether or not to create a DMZ node group with taints"
+  description = "Whether or not to utilise the DMZ node group if it exists"
   default     = false
 }