diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 4131217..86285bc 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -1,72 +1,32 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. -provider "registry.terraform.io/gavinbunney/kubectl" { - version = "1.14.0" - constraints = ">= 1.14.0" - hashes = [ - "h1:Ck8Re/28x7VBI5ArFg0VSg1woPu/APm1ZbMuzqUdnPo=", - "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", - "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", - "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", - "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", - "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", - "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", - "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", - "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", - "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", - "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", - ] -} - provider "registry.terraform.io/hashicorp/aws" { - version = "5.37.0" - constraints = ">= 3.29.0, >= 3.72.0, >= 4.0.0, >= 4.47.0, >= 5.30.0, 5.37.0" + version = "5.60.0" + constraints = ">= 3.29.0, >= 4.0.0, >= 5.30.0, >= 5.60.0, 5.60.0" hashes = [ - "h1:6qJfvyWObjLPoUrEC8kNVAJ1ZFFrIgzC1xprMkkoSjo=", - "h1:WcdVLFBrCN1lP44ZzCSTR8e8p/4C9BQLAqdszE+jh4M=", - "zh:00f40a3d9593476693a7a72d993fd289f7be374fe3f2799776c6296eb6ff890a", - "zh:1010a9fbf55852a8da3473de4ec0f1fcf29efa85d66f61cbe2b086dbbd7747ae", - "zh:103a5674d1eb1cff05fe35e9baa9875afd18d740868b63f9c0c25eadb5eb4eb7", - "zh:270ac1b7a1327c1456a43df44c0b5cc3e26ed6d8861a709adeea1da684a563f5", - "zh:424362c02c8917c0586f3dd49aca27b7e0c21f5a23374b7045e9be3b5646c028", - "zh:549fa2ea187964ab9a0c354310947ead30e09b3199db1ff377c21d7547d78299", - "zh:6492d2ccc7f7d60e83cd8b7244adc53f30efc17d84b1ffc1b8fd6c385f8255fd", - "zh:66fb7b3b8a357071d26c5996c16d426edf07502a05ac86f4a6f73646ee7d1bbb", - "zh:6ecc05fb466d06ea8945564d2cdb8c2a8827d8cfca1550e9fb7eac0e95920196", - "zh:7932360b627b211dad937d278a8692a6c52bd6c0a71e4ec9e94ccbe825053822", - "zh:97ed1b4a18842c4d56a735329e87b4ef91a47e820e5a5c3c2dd64e293408bfc8", + "h1:LohYoaBivwkHi9UuZJzNEIMdDX3WVu7xO/VtGygmonA=", + "zh:08f49c9eb865e136a55dda3eb2b790f6d55cdac49f6638391dbea4b865cf307b", + "zh:090dd8b40ebf0f8e9ea05b9a142add9caeb7988d3d96c5c112e8c67c0edf566f", + "zh:30f336af1b4f0824fce2cc6e81af0986b325b135436c9d892d081e435aeed67e", + "zh:338195ca3b41249874110253412d8913f770c22294af05799ea1e343050906f5", + "zh:3a8a45b17750b01192a0fbeeed0d05c2c04840344d78d5e3233b3ecbeec17a1c", + "zh:486efe72d39f0736d9b7e00e5b889288264458a57aa0cff2d75688d6db372ee5", + "zh:5fdccc448a085fea8ecfae43ae326840abfcdf1a0aa8b8c79dd466392aa5cc3a", + "zh:9521639755cd07ec7efde86a534770e436e16a93692d070a00f6419c1038d59c", "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", - "zh:d5e022052011e1984b9c2f8bc5a6b05c909e3b5bf40c3baddf191bf90e3169c2", - "zh:d7e9488b2ce5904efb91c8577b3fe9b0cd599c4cd508f1f163f292930f54fdf0", - "zh:e57cd93d5cd81dd0f446076af6e47a53ce83df2947ec64ed39a1090d4bdf8f0b", - ] -} - -provider "registry.terraform.io/hashicorp/cloudinit" { - version = "2.3.4" - constraints = ">= 2.0.0" - hashes = [ - "h1:+J2rgfJH5B0vyFR0Wfcoyt4SHWfZLDe+WtUMtmZLDeY=", - "h1:cVIIhnXweOHavu1uV2bdKScTjLbM1WnKM/25wqYBJWo=", - "zh:09f1f1e1d232da96fbf9513b0fb5263bc2fe9bee85697aa15d40bb93835efbeb", - "zh:381e74b90d7a038c3a8dcdcc2ce8c72d6b86da9f208a27f4b98cabe1a1032773", - "zh:398eb321949e28c4c5f7c52e9b1f922a10d0b2b073b7db04cb69318d24ffc5a9", - "zh:4a425679614a8f0fe440845828794e609b35af17db59134c4f9e56d61e979813", - "zh:4d955d8608ece4984c9f1dacda2a59fdb4ea6b0243872f049b388181aab8c80a", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:a48fbee1d58d55a1f4c92c2f38c83a37c8b2f2701ed1a3c926cefb0801fa446a", - "zh:b748fe6631b16a1dafd35a09377c3bffa89552af584cf95f47568b6cd31fc241", - "zh:d4b931f7a54603fa4692a2ec6e498b95464babd2be072bed5c7c2e140a280d99", - "zh:f1c9337fcfe3a7be39d179eb7986c22a979cfb2c587c05f1b3b83064f41785c5", - "zh:f58fc57edd1ee3250a28943cd84de3e4b744cdb52df0356a53403fc240240636", - "zh:f5f50de0923ff530b03e1bca0ac697534d61bb3e5fc7f60e13becb62229097a9", + "zh:c2fb9240a069da9f51e7379e76c3dfaad15a97430c2e32708a7d18345434e310", + "zh:daba836b89537dfa72bb8c77e88850c20fda2a3d0f5b3803cd3d6da0ce283e3e", + "zh:db7e0755ed120ed8311f6663f49aa7157da5072b906727db3a6c47d64e0b82c6", + "zh:ea5e3fca5197639c4ad1415ca96de2924a351ecd1a885dd9184843d5eec18dbb", + "zh:f3f322951d311e45a47361f24790a90a0b8ba6d3829a00c4066a361960d2ecef", + "zh:f48b44f4887d4b51a1406057f15f1e2161cb02b271b2659349958904c678e91c", ] } provider "registry.terraform.io/hashicorp/helm" { version = "2.13.2" - constraints = ">= 2.4.1" + constraints = "2.13.2" hashes = [ "h1:d6W4C33agkPeyIhfJ6EqN3f1i/0IJc8ow+7HbocjtEI=", "h1:nlSqCo0PajJzjSlx0lXNUq1YcOr8p9b3ahcUUYN2pEg=", @@ -108,7 +68,7 @@ provider "registry.terraform.io/hashicorp/http" { provider "registry.terraform.io/hashicorp/kubernetes" { version = "2.30.0" - constraints = ">= 1.6.1, >= 2.10.0" + constraints = ">= 1.6.1, >= 2.10.0, 2.30.0" hashes = [ "h1:KFBOyOGlS+BGrDfbuVdBhTIRefDo+vJEO/IooUR6T4g=", "h1:wRVWY3sK32BNInDOlQnoGSmL638f3jjLFypCAotwpc8=", @@ -127,51 +87,9 @@ provider "registry.terraform.io/hashicorp/kubernetes" { ] } -provider "registry.terraform.io/hashicorp/local" { - version = "2.5.1" - constraints = ">= 2.1.0" - hashes = [ - "h1:8oTPe2VUL6E2d3OcrvqyjI4Nn/Y/UEQN26WLk5O/B0g=", - "h1:Np4kERf9SMrqUi7DJ1rK3soMK14k49nfgE7l/ipQ5xw=", - "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561", - "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561", - "zh:196bfb49086f22fd4db46033e01655b0e5e036a5582d250412cc690fa7995de5", - "zh:37c92ec084d059d37d6cffdb683ccf68e3a5f8d2eb69dd73c8e43ad003ef8d24", - "zh:4269f01a98513651ad66763c16b268f4c2da76cc892ccfd54b401fff6cc11667", - "zh:51904350b9c728f963eef0c28f1d43e73d010333133eb7f30999a8fb6a0cc3d8", - "zh:73a66611359b83d0c3fcba2984610273f7954002febb8a57242bbb86d967b635", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:7ae387993a92bcc379063229b3cce8af7eaf082dd9306598fcd42352994d2de0", - "zh:9e0f365f807b088646db6e4a8d4b188129d9ebdbcf2568c8ab33bddd1b82c867", - "zh:b5263acbd8ae51c9cbffa79743fbcadcb7908057c87eb22fd9048268056efbc4", - "zh:dfcd88ac5f13c0d04e24be00b686d069b4879cc4add1b7b1a8ae545783d97520", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" - constraints = ">= 3.1.0" - hashes = [ - "h1:m467k2tZ9cdFFgHW7LPBK2GLPH43LC6wc3ppxr8yvoE=", - "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", - ] -} - provider "registry.terraform.io/hashicorp/random" { version = "3.6.2" - constraints = ">= 3.0.0" + constraints = "3.6.2" hashes = [ "h1:5lstwe/L8AZS/CP0lil2nPvmbbjAu8kCaU/ogSGNbxk=", "h1:wmG0QFjQ2OfyPy6BB7mQ57WtoZZGGV07uAPQeDmIrAE=", @@ -191,8 +109,7 @@ provider "registry.terraform.io/hashicorp/random" { } provider "registry.terraform.io/hashicorp/tls" { - version = "4.0.5" - constraints = ">= 3.0.0" + version = "4.0.5" hashes = [ "h1:e4LBdJoZJNOQXPWgOAG0UuPBVhCStu98PieNlqJTmeU=", "h1:jb/Rg9inGYp4t8HtBoETESsQJgdmOHoe1bzzg2uNB3w=", @@ -210,25 +127,3 @@ provider "registry.terraform.io/hashicorp/tls" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } - -provider "registry.terraform.io/terraform-aws-modules/http" { - version = "2.4.1" - constraints = "2.4.1" - hashes = [ - "h1:ZnkXcawrIr611RvZpoDzbtPU7SVFyHym+7p1t+PQh20=", - "h1:bnpMAHU6468QXsTTfe/aOvzjj8hOALU1f+5iOoe0iNQ=", - "zh:0111f54de2a9815ded291f23136d41f3d2731c58ea663a2e8f0fef02d377d697", - "zh:0740152d76f0ccf54f4d0e8e0753739a5233b022acd60b5d2353d248c4c17204", - "zh:569518f46809ec9cdc082b4dfd4e828236eee2b50f87b301d624cfd83b8f5b0d", - "zh:7669f7691de91eec9f381e9a4be81aa4560f050348a86c6ea7804925752a01bb", - "zh:81cd53e796ec806aca2d8e92a2aed9135661e170eeff6cf0418e54f98816cd05", - "zh:82f01abd905090f978b169ac85d7a5952322a5f0f460269dd981b3596652d304", - "zh:9a235610066e0f7e567e69c23a53327271a6fc568b06bf152d8fe6594749ed2b", - "zh:aeabdd8e633d143feb67c52248c85358951321e35b43943aeab577c005abd30a", - "zh:c20d22dba5c79731918e7192bc3d0b364d47e98a74f47d287e6cc66236bc0ed0", - "zh:c4fea2cb18c31ed7723deec5ebaff85d6795bb6b6ed3b954794af064d17a7f9f", - "zh:e21e88b6e7e55b9f29b046730d9928c65a4f181fd5f60a42f1cd41b46a0a938d", - "zh:eddb888a74dea348a0acdfee13a08875bacddde384bd9c28342a534269665568", - "zh:f46d5f1403b8d8dfafab9bdd7129d3080bb62a91ea726f477fd43560887b8c4a", - ] -} diff --git a/k8s-eks-addons.tf b/k8s-eks-addons.tf index 0f77f4c..5264a07 100644 --- a/k8s-eks-addons.tf +++ b/k8s-eks-addons.tf @@ -1,21 +1,21 @@ -module "k8s_eks_addons" { - source = "./modules/k8s_eks_addons" +# module "k8s_eks_addons" { +# source = "./modules/k8s_eks_addons" - ingress_nginx_config = merge(var.ingress_nginx_config, { subnets_ids = local.public_subnets }) - cluster_autoscaler_config = var.cluster_autoscaler_config - coredns_config = var.coredns_config - s3_csi_config = var.s3_csi_config - aws_load_balancer_controller_config = var.aws_load_balancer_controller_config +# ingress_nginx_config = merge(var.ingress_nginx_config, { subnets_ids = local.public_subnets }) +# cluster_autoscaler_config = var.cluster_autoscaler_config +# coredns_config = var.coredns_config +# s3_csi_config = var.s3_csi_config +# aws_load_balancer_controller_config = var.aws_load_balancer_controller_config - addon_context = { - aws_caller_identity_account_id = data.aws_caller_identity.current.account_id - aws_partition_id = data.aws_partition.current.partition - aws_region_name = data.aws_region.current.name - eks_cluster_id = module.eks.eks_cluster_id - eks_cluster_version = module.eks.eks_cluster_version - eks_oidc_issuer_url = replace(module.eks.eks_oidc_issuer_url, "https://", "") - tags = var.tags - } +# addon_context = { +# aws_caller_identity_account_id = data.aws_caller_identity.current.account_id +# aws_partition_id = data.aws_partition.current.partition +# aws_region_name = data.aws_region.current.name +# eks_cluster_id = module.eks.eks_cluster_id +# eks_cluster_version = module.eks.eks_cluster_version +# eks_oidc_issuer_url = replace(module.eks.eks_oidc_issuer_url, "https://", "") +# tags = var.tags +# } - depends_on = [module.eks.eks_cluster_arn, module.vpc] -} +# depends_on = [module.eks.eks_cluster_arn, module.vpc] +# } diff --git a/k8s.tf b/k8s.tf index acbfe3f..27ae034 100644 --- a/k8s.tf +++ b/k8s.tf @@ -1,110 +1,106 @@ module "eks" { - source = "git::https://github.com/aws-ia/terraform-aws-eks-blueprints.git?ref=v4.32.1" - cluster_version = var.kubernetesVersion - cluster_name = var.infrastructurename - vpc_id = local.vpc_id - private_subnet_ids = local.private_subnets - create_eks = true - map_accounts = var.map_accounts - map_users = var.map_users - map_roles = var.map_roles - tags = var.tags - cloudwatch_log_group_kms_key_id = aws_kms_key.kms_key_cloudwatch_log_group.arn - cloudwatch_log_group_retention_in_days = var.cloudwatch_retention - managed_node_groups = merge(local.default_managed_node_pools, var.gpuNodePool ? local.gpu_node_pool : {}, var.ivsGpuNodePool ? local.ivsgpu_node_pool : {}) + source = "./modules/eks" + cluster_version = var.kubernetesVersion + cluster_name = var.infrastructurename + vpc_id = local.vpc_id + subnet_ids = local.private_subnets + map_accounts = var.map_accounts + map_users = var.map_users + map_roles = var.map_roles + tags = var.tags } -data "aws_eks_node_group" "default" { - cluster_name = local.infrastructurename - node_group_name = replace(module.eks.managed_node_groups[0]["default"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") +# data "aws_eks_node_group" "default" { +# cluster_name = local.infrastructurename +# node_group_name = replace(module.eks.managed_node_groups[0]["default"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") -} +# } -data "aws_eks_node_group" "execnodes" { - cluster_name = local.infrastructurename - node_group_name = replace(module.eks.managed_node_groups[0]["execnodes"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") +# data "aws_eks_node_group" "execnodes" { +# cluster_name = local.infrastructurename +# node_group_name = replace(module.eks.managed_node_groups[0]["execnodes"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") -} +# } -data "aws_eks_node_group" "gpuexecnodes" { - count = var.gpuNodePool ? 1 : 0 - cluster_name = local.infrastructurename - node_group_name = replace(module.eks.managed_node_groups[0]["gpuexecnodes"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") -} +# data "aws_eks_node_group" "gpuexecnodes" { +# count = var.gpuNodePool ? 1 : 0 +# cluster_name = local.infrastructurename +# node_group_name = replace(module.eks.managed_node_groups[0]["gpuexecnodes"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") +# } -data "aws_eks_node_group" "gpuivsnodes" { - count = var.ivsGpuNodePool ? 1 : 0 - cluster_name = local.infrastructurename - node_group_name = replace(module.eks.managed_node_groups[0]["gpuivsnodes"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") -} +# data "aws_eks_node_group" "gpuivsnodes" { +# count = var.ivsGpuNodePool ? 1 : 0 +# cluster_name = local.infrastructurename +# node_group_name = replace(module.eks.managed_node_groups[0]["gpuivsnodes"]["managed_nodegroup_id"][0], "${local.infrastructurename}:", "") +# } -resource "aws_autoscaling_group_tag" "default_node-template_resources_ephemeral-storage" { - autoscaling_group_name = data.aws_eks_node_group.default.resources[0].autoscaling_groups[0].name +# resource "aws_autoscaling_group_tag" "default_node-template_resources_ephemeral-storage" { +# autoscaling_group_name = data.aws_eks_node_group.default.resources[0].autoscaling_groups[0].name - tag { - key = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" - value = "${var.linuxNodeDiskSize}G" +# tag { +# key = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" +# value = "${var.linuxNodeDiskSize}G" - propagate_at_launch = true - } -} +# propagate_at_launch = true +# } +# } -resource "aws_autoscaling_group_tag" "execnodes" { - autoscaling_group_name = data.aws_eks_node_group.execnodes.resources[0].autoscaling_groups[0].name +# resource "aws_autoscaling_group_tag" "execnodes" { +# autoscaling_group_name = data.aws_eks_node_group.execnodes.resources[0].autoscaling_groups[0].name - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/purpose" - value = "execution" +# tag { +# key = "k8s.io/cluster-autoscaler/node-template/label/purpose" +# value = "execution" - propagate_at_launch = true - } -} +# propagate_at_launch = true +# } +# } -# see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup -# https://github.com/kubernetes/autoscaler/issues/1869#issuecomment-518530724 -resource "aws_autoscaling_group_tag" "execnodes_node-template_resources_ephemeral-storage" { - autoscaling_group_name = data.aws_eks_node_group.execnodes.resources[0].autoscaling_groups[0].name +# # see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/cloudprovider/aws/README.md#auto-discovery-setup +# # https://github.com/kubernetes/autoscaler/issues/1869#issuecomment-518530724 +# resource "aws_autoscaling_group_tag" "execnodes_node-template_resources_ephemeral-storage" { +# autoscaling_group_name = data.aws_eks_node_group.execnodes.resources[0].autoscaling_groups[0].name - tag { - key = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" - value = "${var.linuxExecutionNodeDiskSize}G" +# tag { +# key = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" +# value = "${var.linuxExecutionNodeDiskSize}G" - propagate_at_launch = true - } -} +# propagate_at_launch = true +# } +# } -resource "aws_autoscaling_group_tag" "gpuexecnodes" { - count = var.gpuNodePool ? 1 : 0 - autoscaling_group_name = data.aws_eks_node_group.gpuexecnodes[0].resources[0].autoscaling_groups[0].name +# resource "aws_autoscaling_group_tag" "gpuexecnodes" { +# count = var.gpuNodePool ? 1 : 0 +# autoscaling_group_name = data.aws_eks_node_group.gpuexecnodes[0].resources[0].autoscaling_groups[0].name - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/purpose" - value = "gpu" +# tag { +# key = "k8s.io/cluster-autoscaler/node-template/label/purpose" +# value = "gpu" - propagate_at_launch = true - } -} +# propagate_at_launch = true +# } +# } -resource "aws_autoscaling_group_tag" "gpuexecnodes_node-template_resources_ephemeral-storage" { - count = var.gpuNodePool ? 1 : 0 - autoscaling_group_name = data.aws_eks_node_group.gpuexecnodes[0].resources[0].autoscaling_groups[0].name +# resource "aws_autoscaling_group_tag" "gpuexecnodes_node-template_resources_ephemeral-storage" { +# count = var.gpuNodePool ? 1 : 0 +# autoscaling_group_name = data.aws_eks_node_group.gpuexecnodes[0].resources[0].autoscaling_groups[0].name - tag { - key = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" - value = "${var.gpuNodeDiskSize}G" +# tag { +# key = "k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage" +# value = "${var.gpuNodeDiskSize}G" - propagate_at_launch = true - } -} +# propagate_at_launch = true +# } +# } -resource "aws_autoscaling_group_tag" "gpuivsnodes" { - count = var.ivsGpuNodePool ? 1 : 0 - autoscaling_group_name = data.aws_eks_node_group.gpuivsnodes[0].resources[0].autoscaling_groups[0].name +# resource "aws_autoscaling_group_tag" "gpuivsnodes" { +# count = var.ivsGpuNodePool ? 1 : 0 +# autoscaling_group_name = data.aws_eks_node_group.gpuivsnodes[0].resources[0].autoscaling_groups[0].name - tag { - key = "k8s.io/cluster-autoscaler/node-template/label/purpose" - value = "gpu" +# tag { +# key = "k8s.io/cluster-autoscaler/node-template/label/purpose" +# value = "gpu" - propagate_at_launch = true - } -} +# propagate_at_launch = true +# } +# } diff --git a/modules/eks/cluster-auth.tf b/modules/eks/cluster-auth.tf new file mode 100644 index 0000000..34130c0 --- /dev/null +++ b/modules/eks/cluster-auth.tf @@ -0,0 +1,28 @@ +resource "kubernetes_config_map" "aws_auth" { + metadata { + name = "aws-auth" + namespace = "kube-system" + labels = merge( + { + "app.kubernetes.io/managed-by" = "terraform" + "app.kubernetes.io/created-by" = "terraform" + }, + ) + } + + data = { + mapRoles = yamlencode( + distinct(concat( + # local.managed_node_group_aws_auth_config_map, + var.map_roles, + )) + ) + mapUsers = yamlencode(var.map_users) + mapAccounts = yamlencode(var.map_accounts) + } + + depends_on = [ + aws_eks_cluster.eks, + data.http.eks_cluster_readiness + ] +} diff --git a/modules/eks/cluster.tf b/modules/eks/cluster.tf new file mode 100644 index 0000000..96ca9ca --- /dev/null +++ b/modules/eks/cluster.tf @@ -0,0 +1,43 @@ +resource "aws_eks_cluster" "eks" { + name = var.cluster_name + role_arn = aws_iam_role.cluster_role.arn + version = var.cluster_version + enabled_cluster_log_types = [] + bootstrap_self_managed_addons = false + + vpc_config { + subnet_ids = var.subnet_ids + endpoint_private_access = false + endpoint_public_access = true + public_access_cidrs = ["0.0.0.0/0"] + } + + kubernetes_network_config { + ip_family = "ipv4" + } + + encryption_config { + provider { + key_arn = aws_kms_key.cluster.arn + } + resources = ["secrets"] + + } + access_config { + authentication_mode = "CONFIG_MAP" + bootstrap_cluster_creator_admin_permissions = true + } + tags = var.tags + + + timeouts { + create = var.cluster_timeouts["create"] + update = var.cluster_timeouts["update"] + delete = var.cluster_timeouts["delete"] + } + + depends_on = [ + aws_iam_role_policy_attachment.cluster_role, + # aws_cloudwatch_log_group.cluster + ] +} diff --git a/modules/eks/data.tf b/modules/eks/data.tf new file mode 100644 index 0000000..22deccf --- /dev/null +++ b/modules/eks/data.tf @@ -0,0 +1,143 @@ +data "aws_caller_identity" "current" {} +data "aws_partition" "current" {} +data "aws_region" "current" {} +data "aws_iam_session_context" "current" { + arn = data.aws_caller_identity.current.arn +} +data "aws_eks_cluster" "cluster" { + name = aws_eks_cluster.eks.id +} + +data "http" "eks_cluster_readiness" { + url = join("/", [data.aws_eks_cluster.cluster.endpoint, "healthz"]) + ca_cert_pem = base64decode(data.aws_eks_cluster.cluster.certificate_authority[0].data) + request_timeout_ms = 600000 +} + +data "aws_iam_policy_document" "eks_key" { + statement { + sid = "Allow access for all principals in the account that are authorized" + effect = "Allow" + actions = [ + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + "arn:${data.aws_partition.current.id}:iam::${data.aws_caller_identity.current.account_id}:root" + ] + } + + condition { + test = "StringEquals" + variable = "kms:CallerAccount" + values = [data.aws_caller_identity.current.account_id] + } + + condition { + test = "StringEquals" + variable = "kms:ViaService" + values = ["eks.${data.aws_region.current.name}.amazonaws.com"] + } + } + + statement { + sid = "Allow direct access to key metadata to the account" + effect = "Allow" + actions = [ + "kms:Describe*", + "kms:Get*", + "kms:List*", + "kms:RevokeGrant", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + "arn:${data.aws_partition.current.id}:iam::${data.aws_caller_identity.current.account_id}:root" + ] + } + } + + statement { + sid = "Allow access for Key Administrators" + effect = "Allow" + actions = [ + "kms:*" + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [data.aws_iam_session_context.current.issuer_arn] + } + } + + statement { + sid = "Allow use of the key" + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:DescribeKey", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:ReEncrypt*", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + local.cluster_iam_role_pathed_arn + ] + } + } + + statement { + sid = "Allow attachment of persistent resources" + effect = "Allow" + actions = [ + "kms:CreateGrant", + "kms:ListGrants", + "kms:RevokeGrant", + ] + resources = ["*"] + + principals { + type = "AWS" + identifiers = [ + local.cluster_iam_role_pathed_arn + ] + } + + condition { + test = "Bool" + variable = "kms:GrantIsForAWSResource" + values = ["true"] + } + } +} + +data "aws_iam_policy_document" "assume_role_policy" { + statement { + sid = "EKSClusterAssumeRole" + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["eks.${local.dns_suffix}"] + } + } +} + +data "tls_certificate" "cluster_certificate" { + url = aws_eks_cluster.eks.identity[0].oidc[0].issuer +} diff --git a/modules/eks/iam.tf b/modules/eks/iam.tf new file mode 100644 index 0000000..ff3217d --- /dev/null +++ b/modules/eks/iam.tf @@ -0,0 +1,39 @@ +resource "aws_iam_role" "cluster_role" { + name = local.cluster_iam_role_name + path = null + description = null + + assume_role_policy = data.aws_iam_policy_document.assume_role_policy.json + permissions_boundary = null + force_detach_policies = true + + # # cloudwatch related inline_policy + # dynamic "inline_policy" { + # for_each = [1] + # content { + # name = local.cluster_iam_role_name + + # policy = jsonencode({ + # Version = "2012-10-17" + # Statement = [ + # { + # Action = ["logs:CreateLogGroup"] + # Effect = "Deny" + # Resource = aws_cloudwatch_log_group.cluster.arn + # }, + # ] + # }) + # } + # } + + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "cluster_role" { + for_each = toset([ + "${local.policy_arn_prefix}/AmazonEKSClusterPolicy", + "${local.policy_arn_prefix}/AmazonEKSVPCResourceController", + ]) + policy_arn = each.value + role = aws_iam_role.cluster_role.name +} diff --git a/modules/eks/kms.tf b/modules/eks/kms.tf new file mode 100644 index 0000000..d289735 --- /dev/null +++ b/modules/eks/kms.tf @@ -0,0 +1,12 @@ +resource "aws_kms_key" "cluster" { + description = "${var.cluster_name} EKS cluster secret encryption key" + policy = data.aws_iam_policy_document.eks_key.json + enable_key_rotation = true + deletion_window_in_days = 30 + tags = var.tags +} + +resource "aws_kms_alias" "cluster" { + name = "alias/${var.cluster_name}" + target_key_id = aws_kms_key.cluster.key_id +} diff --git a/modules/eks/locals.tf b/modules/eks/locals.tf new file mode 100644 index 0000000..2e8edbe --- /dev/null +++ b/modules/eks/locals.tf @@ -0,0 +1,8 @@ +locals { + prefix_separator = "-" + dns_suffix = data.aws_partition.current.dns_suffix + cluster_iam_role_name = "${var.cluster_name}-cluster-role" + policy_arn_prefix = "arn:${data.aws_partition.current.partition}:iam::aws:policy" + cluster_encryption_policy_name = "${local.cluster_iam_role_name}-ClusterEncryption" + cluster_iam_role_pathed_arn = "arn:${data.aws_partition.current.id}:iam::${data.aws_caller_identity.current.account_id}:role/${local.cluster_iam_role_name}" +} diff --git a/modules/eks/oidc.tf b/modules/eks/oidc.tf new file mode 100644 index 0000000..4c58581 --- /dev/null +++ b/modules/eks/oidc.tf @@ -0,0 +1,10 @@ +resource "aws_iam_openid_connect_provider" "oidc_provider" { + client_id_list = distinct(compact(concat(["sts.${local.dns_suffix}"], []))) + thumbprint_list = concat([data.tls_certificate.cluster_certificate.certificates[0].sha1_fingerprint], []) + url = aws_eks_cluster.eks.identity[0].oidc[0].issuer + + tags = merge( + { Name = "${var.cluster_name}-eks-irsa" }, + var.tags + ) +} diff --git a/modules/eks/outputs.tf b/modules/eks/outputs.tf new file mode 100644 index 0000000..9f62512 --- /dev/null +++ b/modules/eks/outputs.tf @@ -0,0 +1,24 @@ +output "cluster_primary_security_group_id" { + description = "Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console" + value = aws_eks_cluster.eks.vpc_config[0].cluster_security_group_id +} + +output "eks_cluster_id" { + description = "The name/id of the EKS cluster. Will block on cluster creation until the cluster is really ready" + value = aws_eks_cluster.eks.id +} + +output "eks_oidc_issuer" { + description = "The OpenID Connect identity provider issuer" + value = aws_eks_cluster.eks.identity[0].oidc[0].issuer +} + +output "eks_oidc_issuer_url" { + description = "The URL on the EKS cluster OIDC Issuer" + value = split("//", aws_eks_cluster.eks.identity[0].oidc[0].issuer)[1] +} + +output "eks_oidc_provider_arn" { + description = "The ARN of the OIDC Provider" + value = aws_iam_openid_connect_provider.oidc_provider.arn +} diff --git a/modules/eks/variables.tf b/modules/eks/variables.tf new file mode 100644 index 0000000..09de05a --- /dev/null +++ b/modules/eks/variables.tf @@ -0,0 +1,61 @@ +variable "cluster_name" { + description = "Name of the EKS cluster that will be created" + type = string +} + +variable "cluster_version" { + description = "Kubernetes version of EKS cluster that will be created" + type = string +} + +variable "cluster_timeouts" { + description = "Create, update, and delete timeout configurations for the cluster" + type = map(string) + default = { + "create" : "60m" + "update" : "60m" + "delete" : "60m" + } +} + +variable "vpc_id" { + description = "ID of the VPC in which EKS will be created" + type = string +} + +variable "subnet_ids" { + description = "A list of subnet IDs where the nodes/node groups will be provisioned. If `control_plane_subnet_ids` is not provided, the EKS cluster control plane (ENIs) will be provisioned in these subnets" + type = list(string) +} + +variable "map_accounts" { + description = "Additional AWS account numbers to add to the aws-auth ConfigMap" + type = list(string) + default = [] +} + +variable "map_roles" { + description = "Additional IAM roles to add to the aws-auth ConfigMap" + type = list(object({ + rolearn = string + username = string + groups = list(string) + })) + default = [] +} + +variable "map_users" { + description = "Additional IAM users to add to the aws-auth ConfigMap" + type = list(object({ + userarn = string + username = string + groups = list(string) + })) + default = [] +} + +variable "tags" { + description = "A map of tags to add to all resources" + type = map(string) + default = {} +} diff --git a/modules/eks/versions.tf b/modules/eks/versions.tf new file mode 100644 index 0000000..2b71e12 --- /dev/null +++ b/modules/eks/versions.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 1.3.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.60.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.10" + } + tls = { + source = "hashicorp/tls" + version = "= 4.0.5" + } + http = { + source = "hashicorp/http" + version = "= 3.4.3" + } + } +} diff --git a/storages.tf b/storages.tf index bc7607f..08179e0 100644 --- a/storages.tf +++ b/storages.tf @@ -46,23 +46,23 @@ resource "aws_efs_mount_target" "mount_target" { security_groups = [module.eks.cluster_primary_security_group_id] } -resource "kubernetes_storage_class_v1" "efs" { - metadata { - name = "efs" - } +# resource "kubernetes_storage_class_v1" "efs" { +# metadata { +# name = "efs" +# } - storage_provisioner = "efs.csi.aws.com" - parameters = { - provisioningMode = "efs-ap" # Dynamic provisioning - fileSystemId = aws_efs_file_system.efs_file_system.id - directoryPerms = "700" - } +# storage_provisioner = "efs.csi.aws.com" +# parameters = { +# provisioningMode = "efs-ap" # Dynamic provisioning +# fileSystemId = aws_efs_file_system.efs_file_system.id +# directoryPerms = "700" +# } - mount_options = [ - "iam" - ] +# mount_options = [ +# "iam" +# ] - depends_on = [ - module.k8s_eks_addons - ] -} +# depends_on = [ +# module.k8s_eks_addons +# ] +# } diff --git a/versions.tf b/versions.tf index 6ec0c34..13f91e4 100644 --- a/versions.tf +++ b/versions.tf @@ -4,23 +4,23 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "= 5.37.0" - # minimum version of 5.37.0 is required to enable ECR pull-through functionality. + version = "= 5.60.0" + # minimum version 5.60.0 is required due to argument requirements for the aws_eks_cluster resource. } + kubernetes = { source = "hashicorp/kubernetes" - version = ">= 2.10" + version = "= 2.30.0" } helm = { source = "hashicorp/helm" - version = ">= 2.4.1" + version = "= 2.13.2" } random = { source = "hashicorp/random" - version = ">= 3.0.0" + version = "= 3.6.2" } - } }