diff --git a/README.md b/README.md index 7b75982f..289a19b2 100644 --- a/README.md +++ b/README.md @@ -454,6 +454,7 @@ Encryption is enabled at all AWS resources that are created by Terraform: | Name | Version | |------|---------| | [aws](#provider\_aws) | 5.37.0 | +| [helm](#provider\_helm) | 2.13.2 | | [kubernetes](#provider\_kubernetes) | 2.30.0 | | [random](#provider\_random) | 3.6.2 | @@ -513,6 +514,8 @@ Encryption is enabled at all AWS resources that are created by Terraform: | [aws_ssm_maintenance_window_task.scan](https://registry.terraform.io/providers/hashicorp/aws/5.37.0/docs/resources/ssm_maintenance_window_task) | resource | | [aws_ssm_patch_baseline.production](https://registry.terraform.io/providers/hashicorp/aws/5.37.0/docs/resources/ssm_patch_baseline) | resource | | [aws_ssm_patch_group.patch_group](https://registry.terraform.io/providers/hashicorp/aws/5.37.0/docs/resources/ssm_patch_group) | resource | +| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace_v1.this](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | | [kubernetes_storage_class_v1.efs](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | | [random_string.policy_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [aws_ami.al2gpu_ami](https://registry.terraform.io/providers/hashicorp/aws/5.37.0/docs/data-sources/ami) | data source | @@ -540,7 +543,6 @@ Encryption is enabled at all AWS resources that are created by Terraform: | [codemeter](#input\_codemeter) | Download link for codemeter rpm package. | `string` | `"https://www.wibu.com/support/user/user-software/file/download/13346.html?tx_wibudownloads_downloadlist%5BdirectDownload%5D=directDownload&tx_wibudownloads_downloadlist%5BuseAwsS3%5D=0&cHash=8dba7ab094dec6267346f04fce2a2bcd"` | no | | [ecr\_pullthrough\_cache\_rule\_config](#input\_ecr\_pullthrough\_cache\_rule\_config) | Specifies if ECR pull through cache rule and accompanying resources will be created. Key 'enable' indicates whether pull through cache rule needs to be enabled for the cluster. When 'enable' is set to 'true', key 'exist' indicates whether pull through cache rule already exists for region's private ECR. If key 'enable' is set to 'true', IAM policy will be attached to the cluster's nodes. Additionally, if 'exist' is set to 'false', credentials for upstream registry and pull through cache rule will be created | 
object({
    enable = bool
    exist  = bool
  })
| 
{
  "enable": false,
  "exist": false
}
| no | | [enable\_aws\_for\_fluentbit](#input\_enable\_aws\_for\_fluentbit) | Install FluentBit to send container logs to CloudWatch. | `bool` | `false` | no | -| [enable\_ingress\_nginx](#input\_enable\_ingress\_nginx) | Enable Ingress Nginx add-on | `bool` | `false` | no | | [enable\_ivs](#input\_enable\_ivs) | n/a | `bool` | `false` | no | | [enable\_patching](#input\_enable\_patching) | Scans license server EC2 instance and EKS nodes for updates. Installs patches on license server automatically. EKS nodes need to be updated manually. | `bool` | `false` | no | | [gpuNodeCountMax](#input\_gpuNodeCountMax) | The maximum number of nodes for gpu job execution | `number` | `12` | no | @@ -550,6 +552,7 @@ Encryption is enabled at all AWS resources that are created by Terraform: | [gpuNodeSize](#input\_gpuNodeSize) | The machine size of the nodes for the gpu job execution | `list(string)` | 
[
  "g5.2xlarge"
]
| no | | [gpuNvidiaDriverVersion](#input\_gpuNvidiaDriverVersion) | The NVIDIA driver version for GPU node group. | `string` | `"535.54.03"` | no | | [infrastructurename](#input\_infrastructurename) | The name of the infrastructure. e.g. simphera-infra | `string` | `"simphera"` | no | +| [ingress\_nginx\_config](#input\_ingress\_nginx\_config) | Input configuration for ingress-nginx service deployed with helm release. By setting key 'enabled' to 'true', ingress-nginx service will be deployed. 'helm\_repository' is an URL for the repository of ingress-nginx helm chart, where 'helm\_version' is its respective version of a chart. 'chart\_values' is used for changing default values.yaml of an ingress-nginx chart. | 
object({
    enable          = bool
    helm_repository = string
    helm_version    = string
    chart_values    = map(any)
  })
| 
{
  "chart_values": {
    "controller": {
      "images": {
        "registry": "registry.k8s.io"
      },
      "service": {
        "annotations": {
          "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing"
        }
      }
    }
  },
  "enable": false,
  "helm_repository": "https://kubernetes.github.io/ingress-nginx",
  "helm_version": "4.1.4"
}
| no | | [install\_schedule](#input\_install\_schedule) | 6-field Cron expression describing the install maintenance schedule. Must not overlap with variable scan\_schedule. | `string` | `"cron(0 3 * * ? *)"` | no | | [ivsGpuNodeCountMax](#input\_ivsGpuNodeCountMax) | The maximum number of GPU nodes nodes for IVS jobs | `number` | `2` | no | | [ivsGpuNodeCountMin](#input\_ivsGpuNodeCountMin) | The minimum number of GPU nodes nodes for IVS jobs | `number` | `0` | no | diff --git a/eks-addons-ingress-nginx.tf b/eks-addons-ingress-nginx.tf new file mode 100644 index 00000000..31ccab0a --- /dev/null +++ b/eks-addons-ingress-nginx.tf @@ -0,0 +1,35 @@ +locals { + helm_config = { + namespace = "nginx" + create_namespace = true + } +} + +resource "kubernetes_namespace_v1" "this" { + count = try(local.helm_config.create_namespace, true) && local.helm_config.namespace != "kube-system" ? 1 : 0 + + metadata { + name = local.helm_config.namespace + } +} + +resource "helm_release" "ingress_nginx" { + count = var.ingress_nginx_config.enable ? 1 : 0 + namespace = local.helm_config.namespace + name = "ingress-nginx" + chart = "ingress-nginx" + repository = var.ingress_nginx_config.helm_repository + version = var.ingress_nginx_config.helm_version + description = "The NGINX HelmChart Ingress Controller deployment configuration" + create_namespace = local.helm_config.create_namespace + dependency_update = true + values = [ + templatefile("${path.module}/templates/nginx_values.yaml", { + public_subnets = join(", ", local.public_subnets) + }), + yamlencode(var.ingress_nginx_config.chart_values), + ] + timeout = 1200 + + depends_on = [module.eks.eks_cluster_arn] +} diff --git a/k8s.tf b/k8s.tf index 0663939a..3c2328a9 100644 --- a/k8s.tf +++ b/k8s.tf @@ -28,7 +28,6 @@ module "eks-addons" { enable_aws_load_balancer_controller = false enable_cluster_autoscaler = true enable_aws_for_fluentbit = var.enable_aws_for_fluentbit - enable_ingress_nginx = var.enable_ingress_nginx tags = var.tags aws_for_fluentbit_helm_config = { values = [templatefile("${path.module}/templates/fluentbit_values.yaml", { @@ -39,17 +38,6 @@ module "eks-addons" { dependency_update = true } - ingress_nginx_helm_config = { - values = [templatefile("${path.module}/templates/nginx_values.yaml", { - internal = "false", - scheme = "internet-facing", - public_subnets = join(", ", local.public_subnets) - })] - namespace = "nginx", - create_namespace = true - dependency_update = true - } - cluster_autoscaler_helm_config = var.cluster_autoscaler_helm_config #depends_on = [module.eks.managed_node_groups] } diff --git a/templates/nginx_values.yaml b/templates/nginx_values.yaml index e2904e1b..bb1bc357 100644 --- a/templates/nginx_values.yaml +++ b/templates/nginx_values.yaml @@ -4,11 +4,9 @@ controller: service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '60' service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: 'true' - service.beta.kubernetes.io/aws-load-balancer-scheme: "${scheme}" - service.beta.kubernetes.io/aws-load-balancer-internal: "${internal}" service.beta.kubernetes.io/aws-load-balancer-target-node-labels: kubernetes.io/os=linux - service.beta.kubernetes.io/aws-load-balancer-subnets: "${public_subnets}" service.beta.kubernetes.io/aws-load-balancer-type: "nlb" + service.beta.kubernetes.io/aws-load-balancer-subnets: "${public_subnets}" metrics: enabled: true port: 10254 diff --git a/terraform.json.example b/terraform.json.example index 80c47879..85ebba73 100644 --- a/terraform.json.example +++ b/terraform.json.example @@ -9,7 +9,6 @@ "exist": false }, "enable_aws_for_fluentbit": false, - "enable_ingress_nginx": false, "enable_ivs": false, "enable_patching": false, "gpuNodeCountMax": 12, @@ -21,6 +20,23 @@ ], "gpuNvidiaDriverVersion": "535.54.03", "infrastructurename": "simphera", + "ingress_nginx_config": { + "chart_values": { + "controller": { + "images": { + "registry": "registry.k8s.io" + }, + "service": { + "annotations": { + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing" + } + } + } + }, + "enable": false, + "helm_repository": "https://kubernetes.github.io/ingress-nginx", + "helm_version": "4.1.4" + }, "install_schedule": "cron(0 3 * * ? *)", "ivsGpuNodeCountMax": 2, "ivsGpuNodeCountMin": 0, diff --git a/terraform.tfvars.example b/terraform.tfvars.example index 8b64c4ed..cc39618c 100644 --- a/terraform.tfvars.example +++ b/terraform.tfvars.example @@ -23,9 +23,6 @@ ecr_pullthrough_cache_rule_config = { # Install FluentBit to send container logs to CloudWatch. enable_aws_for_fluentbit = false -# Enable Ingress Nginx add-on -enable_ingress_nginx = false - enable_ivs = false # Scans license server EC2 instance and EKS nodes for updates. @@ -56,6 +53,28 @@ gpuNvidiaDriverVersion = "535.54.03" # The name of the infrastructure. e.g. simphera-infra infrastructurename = "simphera" +# Input configuration for ingress-nginx service deployed with helm release. +# By setting key 'enabled' to 'true', ingress-nginx service will be deployed. +# 'helm_repository' is an URL for the repository of ingress-nginx helm chart, where 'helm_version' is its respective version of a chart. +# 'chart_values' is used for changing default values.yaml of an ingress-nginx chart. +ingress_nginx_config = { + "chart_values": { + "controller": { + "images": { + "registry": "registry.k8s.io" + }, + "service": { + "annotations": { + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing" + } + } + } + }, + "enable": false, + "helm_repository": "https://kubernetes.github.io/ingress-nginx", + "helm_version": "4.1.4" +} + # 6-field Cron expression describing the install maintenance schedule. Must not overlap with variable scan_schedule. install_schedule = "cron(0 3 * * ? *)" diff --git a/variables.tf b/variables.tf index 4a1ca072..8fc8c0cf 100644 --- a/variables.tf +++ b/variables.tf @@ -208,12 +208,6 @@ variable "rtMaps_link" { default = "http://dl.intempora.com/RTMaps4/rtmaps_4.9.0_ubuntu1804_x86_64_release.tar.bz2" } -variable "enable_ingress_nginx" { - type = bool - description = "Enable Ingress Nginx add-on" - default = false -} - variable "map_accounts" { type = list(string) description = "Additional AWS account numbers to add to the aws-auth ConfigMap" @@ -239,7 +233,32 @@ variable "map_users" { description = "Additional IAM users to add to the aws-auth ConfigMap" default = [] } - +variable "ingress_nginx_config" { + type = object({ + enable = bool + helm_repository = string + helm_version = string + chart_values = map(any) + }) + description = "Input configuration for ingress-nginx service deployed with helm release. By setting key 'enabled' to 'true', ingress-nginx service will be deployed. 'helm_repository' is an URL for the repository of ingress-nginx helm chart, where 'helm_version' is its respective version of a chart. 'chart_values' is used for changing default values.yaml of an ingress-nginx chart." + default = { + enable = false + helm_repository = "https://kubernetes.github.io/ingress-nginx" + helm_version = "4.1.4" + chart_values = { + controller = { + images = { + registry = "registry.k8s.io" + } + service = { + annotations = { + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + } + } + } + } + } +} variable "simpheraInstances" { type = map(object({ name = string