diff --git a/README.md b/README.md index 4590c00..ae3a3a2 100644 --- a/README.md +++ b/README.md @@ -268,7 +268,7 @@ worker_groups = { | [enable\_sso\_rbac](#input\_enable\_sso\_rbac) | Enable SSO RBAC integration or not | `bool` | `false` | no | | [enable\_waf\_for\_alb](#input\_enable\_waf\_for\_alb) | Enables WAF and WAF V2 addons for ALB | `bool` | `false` | no | | [external\_secrets\_namespace](#input\_external\_secrets\_namespace) | The namespace of external-secret operator | `string` | `"kube-system"` | no | -| [fluent\_bit\_configs](#input\_fluent\_bit\_configs) | Fluent Bit configs | 
object({
    fluent_bit_name       = optional(string, "")
    log_group_name        = optional(string, "")
    system_log_group_name = optional(string, "")
    log_retention_days    = optional(number, 90)
    values_yaml           = optional(string, "")
    configs = optional(object({
      inputs                     = optional(string, "")
      filters                    = optional(string, "")
      outputs                    = optional(string, "")
      cloudwatch_outputs_enabled = optional(bool, true)
    }), {})
    drop_namespaces        = optional(list(string), [])
    log_filters            = optional(list(string), [])
    additional_log_filters = optional(list(string), [])
    kube_namespaces        = optional(list(string), [])
  })
| 
{
  "additional_log_filters": [
    "ELB-HealthChecker",
    "Amazon-Route53-Health-Check-Service"
  ],
  "configs": {
    "cloudwatch_outputs_enabled": true,
    "filters": "",
    "inputs": "",
    "outputs": ""
  },
  "drop_namespaces": [
    "kube-system",
    "opentelemetry-operator-system",
    "adot",
    "cert-manager",
    "opentelemetry.*",
    "meta.*"
  ],
  "fluent_bit_name": "",
  "kube_namespaces": [
    "kube.*",
    "meta.*",
    "adot.*",
    "devops.*",
    "cert-manager.*",
    "git.*",
    "opentelemetry.*",
    "stakater.*",
    "renovate.*"
  ],
  "log_filters": [
    "kube-probe",
    "health",
    "prometheus",
    "liveness"
  ],
  "log_group_name": "",
  "log_retention_days": 90,
  "system_log_group_name": "",
  "values_yaml": ""
}
| no | +| [fluent\_bit\_configs](#input\_fluent\_bit\_configs) | Fluent Bit configs | 
object({
    fluent_bit_name       = optional(string, "")
    log_group_name        = optional(string, "")
    system_log_group_name = optional(string, "")
    log_retention_days    = optional(number, 90)
    values_yaml           = optional(string, "")
    configs = optional(object({
      inputs                     = optional(string, "")
      filters                    = optional(string, "")
      outputs                    = optional(string, "")
      cloudwatch_outputs_enabled = optional(bool, true)
    }), {})
    drop_namespaces        = optional(list(string), [])
    log_filters            = optional(list(string), [])
    additional_log_filters = optional(list(string), [])
    kube_namespaces        = optional(list(string), [])
    image_pull_secrets     = optional(list(string), [])
  })
| 
{
  "additional_log_filters": [
    "ELB-HealthChecker",
    "Amazon-Route53-Health-Check-Service"
  ],
  "configs": {
    "cloudwatch_outputs_enabled": true,
    "filters": "",
    "inputs": "",
    "outputs": ""
  },
  "drop_namespaces": [
    "kube-system",
    "opentelemetry-operator-system",
    "adot",
    "cert-manager",
    "opentelemetry.*",
    "meta.*"
  ],
  "fluent_bit_name": "",
  "image_pull_secrets": [],
  "kube_namespaces": [
    "kube.*",
    "meta.*",
    "adot.*",
    "devops.*",
    "cert-manager.*",
    "git.*",
    "opentelemetry.*",
    "stakater.*",
    "renovate.*"
  ],
  "log_filters": [
    "kube-probe",
    "health",
    "prometheus",
    "liveness"
  ],
  "log_group_name": "",
  "log_retention_days": 90,
  "system_log_group_name": "",
  "values_yaml": ""
}
| no | | [manage\_aws\_auth](#input\_manage\_aws\_auth) | n/a | `bool` | `true` | no | | [map\_roles](#input\_map\_roles) | Additional IAM roles to add to the aws-auth configmap. | 
list(object({
    rolearn  = string
    username = string
    groups   = list(string)
  }))
| `[]` | no | | [metrics\_exporter](#input\_metrics\_exporter) | Metrics Exporter, can use cloudwatch or adot | `string` | `"adot"` | no | diff --git a/fluent-bit.tf b/fluent-bit.tf index c6fb423..74aa094 100644 --- a/fluent-bit.tf +++ b/fluent-bit.tf @@ -14,6 +14,7 @@ module "fluent-bit" { log_group_name = try(var.fluent_bit_configs.log_group_name, "") != "" ? var.fluent_bit_configs.log_group_name : "fluent-bit-cloudwatch-${module.eks-cluster[0].cluster_id}" system_log_group_name = try(var.fluent_bit_configs.system_log_group_name, "") log_retention_days = try(var.fluent_bit_configs.log_retention_days, 90) + image_pull_secrets = try(var.fluent_bit_configs.image_pull_secrets, []) values_yaml = try(var.fluent_bit_configs.values_yaml, "") diff --git a/modules/fluent-bit/README.md b/modules/fluent-bit/README.md index 10d2fbf..bcce4c6 100644 --- a/modules/fluent-bit/README.md +++ b/modules/fluent-bit/README.md @@ -55,6 +55,7 @@ No modules. | [eks\_oidc\_root\_ca\_thumbprint](#input\_eks\_oidc\_root\_ca\_thumbprint) | n/a | `string` | n/a | yes | | [fluent\_bit\_config](#input\_fluent\_bit\_config) | You can add other inputs,outputs and filters which module doesn't have by default | `any` | 
{
  "cloudwatch_outputs_enabled": true,
  "filters": "",
  "inputs": "",
  "outputs": ""
}
| no | | [fluent\_bit\_name](#input\_fluent\_bit\_name) | Container resource name. | `string` | `"fluent-bit"` | no | +| [image\_pull\_secrets](#input\_image\_pull\_secrets) | Secret name which can we use for download image | `list(string)` | `[]` | no | | [kube\_namespaces](#input\_kube\_namespaces) | Kubernates namespaces | `list(string)` | 
[
  "kube.*",
  "meta.*",
  "adot.*",
  "devops.*",
  "cert-manager.*",
  "git.*",
  "opentelemetry.*",
  "stakater.*",
  "renovate.*"
]
| no | | [log\_filters](#input\_log\_filters) | Fluent bit doesn't send logs if message consists of this values | `list(string)` | 
[
  "kube-probe",
  "health",
  "prometheus",
  "liveness"
]
| no | | [log\_group\_name](#input\_log\_group\_name) | Log group name fluent-bit will be streaming logs into. | `string` | `"fluentbit-default-log-group"` | no | diff --git a/modules/fluent-bit/locals.tf b/modules/fluent-bit/locals.tf index 876ec2e..c1f9201 100644 --- a/modules/fluent-bit/locals.tf +++ b/modules/fluent-bit/locals.tf @@ -16,6 +16,7 @@ locals { filters = try(var.fluent_bit_config.filters, "") cloudwatch_outputs_enabled = try(var.fluent_bit_config.cloudwatch_outputs_enabled, true) kube_namespaces = var.kube_namespaces + imagePullSecrets = [for item in var.image_pull_secrets : { name : item }] } values = var.values_yaml == "" ? templatefile("${path.module}/values.yaml.tpl", local.config_settings) : var.values_yaml diff --git a/modules/fluent-bit/values.yaml.tpl b/modules/fluent-bit/values.yaml.tpl index 6fc956c..1947f4e 100644 --- a/modules/fluent-bit/values.yaml.tpl +++ b/modules/fluent-bit/values.yaml.tpl @@ -1,3 +1,4 @@ +imagePullSecrets: ${jsonencode(imagePullSecrets)} config: ## https://docs.fluentbit.io/manual/pipeline/inputs inputs: | diff --git a/modules/fluent-bit/variables.tf b/modules/fluent-bit/variables.tf index 964e029..70ffae5 100644 --- a/modules/fluent-bit/variables.tf +++ b/modules/fluent-bit/variables.tf @@ -138,3 +138,9 @@ variable "additional_log_filters" { ] description = "Fluent bit doesn't send logs if message consists of this values" } + +variable "image_pull_secrets" { + type = list(string) + default = [] + description = "Secret name which can we use for download image" +} diff --git a/tests/eks-fluent-bit/0-setup.tf b/tests/eks-fluent-bit/0-setup.tf index ca00286..1b4b9d4 100644 --- a/tests/eks-fluent-bit/0-setup.tf +++ b/tests/eks-fluent-bit/0-setup.tf @@ -1,16 +1,12 @@ terraform { required_providers { - test = { - source = "terraform.io/builtin/test" - } - aws = { source = "hashicorp/aws" version = ">= 3.41" } } - required_version = ">= 1.3.0, < 1.6.0" + required_version = ">= 1.3.0, < 2.0.0" } /** diff --git a/tests/eks-fluent-bit/1-example.tf b/tests/eks-fluent-bit/1-example.tf index b570ee1..13afa6f 100644 --- a/tests/eks-fluent-bit/1-example.tf +++ b/tests/eks-fluent-bit/1-example.tf @@ -12,22 +12,21 @@ data "aws_subnet_ids" "subnets" { module "this" { source = "../.." - account_id = "0000000000" adot_config = { "accept_namespace_regex" : "(default|kube-system)", "additional_metrics" : [], "log_group_name" : "adot-logs" } cluster_enabled_log_types = ["audit"] - cluster_name = "eks-dev" + cluster_name = "test-eks-fluent-bit" cluster_version = "1.27" metrics_exporter = "adot" node_groups = { "dev_nodes" : { - "desired_size" : 2, - "max_capacity" : 5, - "max_size" : 5, - "min_size" : 2 + "desired_size" : 1, + "max_capacity" : 1, + "max_size" : 1, + "min_size" : 1 } } node_groups_default = { @@ -35,10 +34,6 @@ module "this" { "instance_types" : ["t3.medium"] } send_alb_logs_to_cloudwatch = false - users = [ - { "username" : "dasmeta" }, - ] - vpc = { link = { id = data.aws_vpcs.ids.ids[0] @@ -47,10 +42,11 @@ module "this" { } fluent_bit_configs = { - config = { + configs = { inputs = templatefile("${path.module}/templates/inputs.yaml.tpl", {}) outputs = templatefile("${path.module}/templates/outputs.yaml.tpl", {}) filters = templatefile("${path.module}/templates/filters.yaml.tpl", {}) + # cloudwatch_outputs_enabled = false # uncomment in case you want also to disable default cloudwatch log exporters/outputs } drop_namespaces = [ "kube-system", diff --git a/tests/eks-fluent-bit/2-assert.tf b/tests/eks-fluent-bit/2-assert.tf deleted file mode 100644 index 99458ca..0000000 --- a/tests/eks-fluent-bit/2-assert.tf +++ /dev/null @@ -1,9 +0,0 @@ -resource "test_assertions" "dummy" { - component = "this" - - equal "scheme" { - description = "As module does not have any output and data just make sure the case runs. Probably can be thrown away." - got = "all good" - want = "all good" - } -} diff --git a/tests/eks-fluent-bit/README.md b/tests/eks-fluent-bit/README.md index e8ab3f4..ee3b419 100644 --- a/tests/eks-fluent-bit/README.md +++ b/tests/eks-fluent-bit/README.md @@ -5,15 +5,14 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0, < 1.6.0 | +| [terraform](#requirement\_terraform) | >= 1.3.0, < 2.0.0 | | [aws](#requirement\_aws) | >= 3.41 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 3.41 | -| [test](#provider\_test) | n/a | +| [aws](#provider\_aws) | 4.67.0 | ## Modules @@ -25,7 +24,6 @@ | Name | Type | |------|------| -| test_assertions.dummy | resource | | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_subnet_ids.subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet_ids) | data source | | [aws_vpcs.ids](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpcs) | data source | diff --git a/tests/eks-fluent-bit/templates/outputs.yaml.tpl b/tests/eks-fluent-bit/templates/outputs.yaml.tpl index 4c1378a..ab5a602 100644 --- a/tests/eks-fluent-bit/templates/outputs.yaml.tpl +++ b/tests/eks-fluent-bit/templates/outputs.yaml.tpl @@ -1,7 +1,7 @@ [OUTPUT] Name s3 Match test.* - bucket s3-bucket + bucket test-eks-fluent-bit-dasmeta region eu-central-1 total_file_size 250M s3_key_format /%Y/%m/%d/%H_%M_%S.gz diff --git a/variables.tf b/variables.tf index d377ed5..a1008a7 100644 --- a/variables.tf +++ b/variables.tf @@ -121,6 +121,7 @@ variable "fluent_bit_configs" { log_filters = optional(list(string), []) additional_log_filters = optional(list(string), []) kube_namespaces = optional(list(string), []) + image_pull_secrets = optional(list(string), []) }) default = { fluent_bit_name = "" @@ -128,6 +129,7 @@ variable "fluent_bit_configs" { system_log_group_name = "" log_retention_days = 90 values_yaml = "" + image_pull_secrets = [] configs = { inputs = "" outputs = ""