From b6db4f2e0246b0dd0e5d2157ab14525cd0ce9bf5 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 2 Apr 2025 13:59:25 +0100 Subject: [PATCH 01/21] feat: move logs agents to its own tile --- .catalog-onboard-pipeline.yaml | 13 + .github/settings.yml | 2 +- .releaserc | 3 + .secrets.baseline | 15 +- CODEOWNERS | 2 + README.md | 183 +++++--------- cra-config.yaml | 19 +- cra-tf-validate-ignore-rules.json | 22 +- examples/advanced/README.md | 4 - examples/advanced/main.tf | 32 --- examples/advanced/outputs.tf | 38 --- examples/advanced/variables.tf | 39 --- examples/advanced/version.tf | 16 -- examples/basic/README.md | 11 - examples/basic/main.tf | 31 --- examples/basic/outputs.tf | 38 --- examples/basic/provider.tf | 8 - examples/basic/variables.tf | 39 --- examples/basic/version.tf | 16 -- examples/logs-agent-iks/README.md | 11 + examples/logs-agent-iks/main.tf | 156 ++++++++++++ examples/logs-agent-iks/outputs.tf | 11 + examples/logs-agent-iks/provider.tf | 24 ++ examples/logs-agent-iks/variables.tf | 41 +++ examples/logs-agent-iks/version.tf | 26 ++ examples/logs-agent-ocp/README.md | 13 + examples/logs-agent-ocp/main.tf | 203 +++++++++++++++ examples/logs-agent-ocp/outputs.tf | 12 + examples/logs-agent-ocp/provider.tf | 24 ++ examples/logs-agent-ocp/variables.tf | 47 ++++ examples/logs-agent-ocp/version.tf | 21 ++ ibm_catalog.json | 215 ++++++++++++++++ images/logs-icon.svg | 1 + kubeconfig/.gitignore | 6 + kubeconfig/README.md | 2 + main.tf | 144 ++++++++++- outputs.tf | 35 +-- .../deployable-architecture-logs-agents.svg | 4 + renovate.json | 13 +- scripts/confirm-rollout-status.sh | 8 + solutions/fully-configurable/DA-types.md | 110 ++++++++ solutions/fully-configurable/README.md | 11 + .../catalogValidationValues.json.template | 3 + .../fully-configurable/kubeconfig/.gitignore | 6 + .../fully-configurable/kubeconfig/README.md | 2 + solutions/fully-configurable/main.tf | 45 ++++ .../fully-configurable/outputs.tf | 8 +- solutions/fully-configurable/provider.tf | 37 +++ solutions/fully-configurable/variables.tf | 211 ++++++++++++++++ solutions/fully-configurable/version.tf | 20 ++ tests/other_test.go | 21 ++ tests/pr_test.go | 238 ++++++++++++++++-- tests/resources/README.md | 1 + tests/resources/main.tf | 139 ++++++++++ tests/resources/outputs.tf | 48 ++++ tests/resources/provider.tf | 4 + tests/resources/variables.tf | 29 +++ tests/resources/version.tf | 10 + ...ation-deploy-slz-roks-and-obs-instances.sh | 20 ++ ...ation-deploy-slz-roks-and-obs-instances.sh | 55 ++++ variables.tf | 233 +++++++++++++++-- version.tf | 17 +- 62 files changed, 2300 insertions(+), 516 deletions(-) create mode 100644 .catalog-onboard-pipeline.yaml create mode 100644 CODEOWNERS delete mode 100644 examples/advanced/README.md delete mode 100644 examples/advanced/main.tf delete mode 100644 examples/advanced/outputs.tf delete mode 100644 examples/advanced/variables.tf delete mode 100644 examples/advanced/version.tf delete mode 100644 examples/basic/README.md delete mode 100644 examples/basic/main.tf delete mode 100644 examples/basic/outputs.tf delete mode 100644 examples/basic/provider.tf delete mode 100644 examples/basic/variables.tf delete mode 100644 examples/basic/version.tf create mode 100644 examples/logs-agent-iks/README.md create mode 100644 examples/logs-agent-iks/main.tf create mode 100644 examples/logs-agent-iks/outputs.tf create mode 100644 examples/logs-agent-iks/provider.tf create mode 100644 examples/logs-agent-iks/variables.tf create mode 100644 examples/logs-agent-iks/version.tf create mode 100644 examples/logs-agent-ocp/README.md create mode 100644 examples/logs-agent-ocp/main.tf create mode 100644 examples/logs-agent-ocp/outputs.tf create mode 100644 examples/logs-agent-ocp/provider.tf create mode 100644 examples/logs-agent-ocp/variables.tf create mode 100644 examples/logs-agent-ocp/version.tf create mode 100644 ibm_catalog.json create mode 100644 images/logs-icon.svg create mode 100644 kubeconfig/.gitignore create mode 100644 kubeconfig/README.md create mode 100644 reference-architecture/deployable-architecture-logs-agents.svg create mode 100755 scripts/confirm-rollout-status.sh create mode 100644 solutions/fully-configurable/DA-types.md create mode 100644 solutions/fully-configurable/README.md create mode 100644 solutions/fully-configurable/catalogValidationValues.json.template create mode 100644 solutions/fully-configurable/kubeconfig/.gitignore create mode 100644 solutions/fully-configurable/kubeconfig/README.md create mode 100644 solutions/fully-configurable/main.tf rename examples/advanced/provider.tf => solutions/fully-configurable/outputs.tf (58%) create mode 100644 solutions/fully-configurable/provider.tf create mode 100644 solutions/fully-configurable/variables.tf create mode 100644 solutions/fully-configurable/version.tf create mode 100644 tests/resources/README.md create mode 100644 tests/resources/main.tf create mode 100644 tests/resources/outputs.tf create mode 100644 tests/resources/provider.tf create mode 100644 tests/resources/variables.tf create mode 100644 tests/resources/version.tf create mode 100644 tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh create mode 100644 tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml new file mode 100644 index 00000000..a9633bcb --- /dev/null +++ b/.catalog-onboard-pipeline.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +offerings: + - name: deploy-arch-ibm-logs-agent + kind: solution + catalog_id: 7df1e4ca-d54c-4fd0-82ce-3d13247308cd + offering_id: _ + variations: + - name: fully-configurable + mark_ready: true + install_type: fullstack + pre_validation: "tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh" + post_validation: "tests/scripts/post-validation-destroy-slz-roks-and-obs-instances.sh" diff --git a/.github/settings.yml b/.github/settings.yml index f72dd194..f0f5f18b 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -25,4 +25,4 @@ repository: # description: "" # Use a comma-separated list of topics to set on the repo (ensure not to use any caps in the topic string). - topics: terraform, ibm-cloud, terraform-module, core-team, logs-agent, observability, agent, cloud-logs, logging-agent + topics: terraform, ibm-cloud, terraform-module, core-team, logs-agent, observability, logs, agent, cloud-logs, logging-agent diff --git a/.releaserc b/.releaserc index 708916f7..622ce915 100644 --- a/.releaserc +++ b/.releaserc @@ -10,6 +10,9 @@ }], ["@semantic-release/exec", { "successCmd": "echo \"SEMVER_VERSION=${nextRelease.version}\" >> $GITHUB_ENV" + }], + ["@semantic-release/exec", { + "publishCmd": "./ci/trigger-catalog-onboarding-pipeline.sh --version=v${nextRelease.version}" }] ] } diff --git a/.secrets.baseline b/.secrets.baseline index 0a4efa4a..7ae5bf9e 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "go.sum|^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-22T17:36:38Z", + "generated_at": "2025-03-13T13:24:51Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -76,18 +76,7 @@ "name": "TwilioKeyDetector" } ], - "results": { - "README.md": [ - { - "hashed_secret": "ff9ee043d85595eb255c05dfe32ece02a53efbb2", - "is_secret": false, - "is_verified": false, - "line_number": 74, - "type": "Secret Keyword", - "verified_result": null - } - ] - }, + "results": {}, "version": "0.13.1+ibm.62.dss", "word_list": { "file": null, diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..dcdf1840 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,2 @@ +# Primary owner should be listed first in list of global owners, followed by any secondary owners +* @Aashiq-J @shemau diff --git a/README.md b/README.md index 26b296aa..d93efb62 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ - -# Terraform modules template project +# Logs Agent module - -TODO: Replace this with a description of the modules in this repo. - +You can use this module to deploy Logs Agent in your cluster. The agent can collect and route application logs from a Red Hat OpenShift or Kubernetes cluster to an IBM Cloud Logs of your choice. - - -## Overview -* [terraform-ibm-logs-agent](#terraform-ibm-logs-agent) -* [Examples](./examples) - * [Advanced example](./examples/advanced) - * [Basic example](./examples/basic) -* [Contributing](#contributing) - - - - - - - - -## terraform-ibm-logs-agent - -### Usage - - +## Usage ```hcl -terraform { - required_version = ">= 1.9.0" - required_providers { - ibm = { - source = "IBM-Cloud/ibm" - version = "X.Y.Z" # Lock into a provider version that satisfies the module constraints - } - } -} +# ############################################################################ +# Init cluster config for helm +# ############################################################################ -locals { - region = "us-south" +data "ibm_container_cluster_config" "cluster_config" { + # update this value with the Id of the cluster where the agent will be provisioned + cluster_name_id = "cluster_id" } +# ############################################################################ +# Config providers +# ############################################################################ + provider "ibm" { - ibmcloud_api_key = "XXXXXXXXXX" # replace with apikey value - region = local.region + # update this value with your IBM Cloud API key value + ibmcloud_api_key = "api key value" # pragma: allowlist secret } -module "module_template" { - source = "terraform-ibm-modules//ibm" - version = "X.Y.Z" # Replace "X.Y.Z" with a release version to lock into a specific release - region = local.region - name = "instance-name" - resource_group_id = "xxXXxxXXxXxXXXXxxXxxxXXXXxXXXXX" # Replace with the actual ID of resource group to use +provider "helm" { + kubernetes { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate + } } -``` - -### Required access policies - - - - - - - - +# ############################################################################ +# Install Logs Agent +# ############################################################################ + +module "logs_agent_module" { + source = "terraform-ibm-modules/logs-agent/ibm" + # update this with your cluster id where the agent will be installed + cluster_id = "cluster id" + # update this with the Id of your IBM Cloud resource group + cluster_resource_group_id = "resource group id" + # Logs Agent variables + logs_agent_trusted_profile = "XXXXXXXX" + cloud_logs_ingress_endpoint = ".ingress.us-south.logs.cloud.ibm.com" + cloud_logs_ingress_port = 443 +} +``` - ### Requirements | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.9.0 | -| [ibm](#requirement\_ibm) | >= 1.71.2, < 2.0.0 | +| [helm](#requirement\_helm) | >= 2.15.0, <3.0.0 | +| [ibm](#requirement\_ibm) | >= 1.76.1, <2.0.0 | ### Modules @@ -133,30 +76,42 @@ No modules. | Name | Type | |------|------| -| [ibm_resource_instance.cos_instance](https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/resources/resource_instance) | resource | +| [helm_release.logs_agent](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [ibm_container_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster) | data source | +| [ibm_container_cluster_config.cluster_config](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_cluster_config) | data source | +| [ibm_container_vpc_cluster.cluster](https://registry.terraform.io/providers/ibm-cloud/ibm/latest/docs/data-sources/container_vpc_cluster) | data source | ### Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [name](#input\_name) | A descriptive name used to identify the resource instance. | `string` | n/a | yes | -| [plan](#input\_plan) | The name of the plan type supported by service. | `string` | `"standard"` | no | -| [resource\_group\_id](#input\_resource\_group\_id) | The ID of the resource group where you want to create the service. | `string` | n/a | yes | -| [resource\_tags](#input\_resource\_tags) | List of resource tag to associate with the instance. | `list(string)` | `[]` | no | +| [cloud\_logs\_ingress\_endpoint](#input\_cloud\_logs\_ingress\_endpoint) | The host for IBM Cloud Logs ingestion. Ensure you use the ingress endpoint. See https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-endpoints_ingress. | `string` | n/a | yes | +| [cloud\_logs\_ingress\_port](#input\_cloud\_logs\_ingress\_port) | The target port for the IBM Cloud Logs ingestion endpoint. The port must be 443 if you connect by using a VPE gateway, or port 3443 when you connect by using CSEs. | `number` | `3443` | no | +| [cluster\_config\_endpoint\_type](#input\_cluster\_config\_endpoint\_type) | The type of endpoint to use for the cluster config access: `default`, `private`, `vpe`, or `link`. The `default` value uses the default endpoint of the cluster. | `string` | `"default"` | no | +| [cluster\_id](#input\_cluster\_id) | The ID of the cluster to deploy the agent. | `string` | n/a | yes | +| [cluster\_resource\_group\_id](#input\_cluster\_resource\_group\_id) | The resource group ID of the cluster. | `string` | n/a | yes | +| [is\_vpc\_cluster](#input\_is\_vpc\_cluster) | Specify true if the target cluster for the agent is a VPC cluster, false if it is a classic cluster. | `bool` | `true` | no | +| [logs\_agent\_additional\_log\_source\_paths](#input\_logs\_agent\_additional\_log\_source\_paths) | The list of additional log sources. By default, the Logs agent collects logs from a single source at `/var/log/containers/*.log`. | `list(string)` | `[]` | no | +| [logs\_agent\_additional\_metadata](#input\_logs\_agent\_additional\_metadata) | The list of additional metadata fields to add to the routed logs. | 
list(object({
    key   = optional(string)
    value = optional(string)
  }))
| `[]` | no | +| [logs\_agent\_chart\_location](#input\_logs\_agent\_chart\_location) | The location of the Helm chart for the Logs agent. | `string` | `"oci://icr.io/ibm/observe/logs-agent-helm"` | no | +| [logs\_agent\_chart\_version](#input\_logs\_agent\_chart\_version) | The version of the helm chart to deploy. | `string` | `"1.4.2"` | no | +| [logs\_agent\_enable\_scc](#input\_logs\_agent\_enable\_scc) | Whether to enable creation of Security Context Constraints in Openshift. When installing on an OpenShift cluster, this setting is mandatory to configure permissions for pods within your cluster. | `bool` | `true` | no | +| [logs\_agent\_exclude\_log\_source\_paths](#input\_logs\_agent\_exclude\_log\_source\_paths) | The list of log sources to exclude. Specify the paths that the Logs agent ignores. | `list(string)` | `[]` | no | +| [logs\_agent\_iam\_api\_key](#input\_logs\_agent\_iam\_api\_key) | The IBM Cloud API key for the Logs agent to authenticate and communicate with the IBM Cloud Logs. It is required if `logs_agent_iam_mode` is set to `IAMAPIKey`. | `string` | `null` | no | +| [logs\_agent\_iam\_environment](#input\_logs\_agent\_iam\_environment) | IAM authentication Environment: `Production` or `PrivateProduction` or `Staging` or `PrivateStaging`. `Production` specifies the public endpoint & `PrivateProduction` specifies the private endpoint. | `string` | `"PrivateProduction"` | no | +| [logs\_agent\_iam\_mode](#input\_logs\_agent\_iam\_mode) | IAM authentication mode: `TrustedProfile` or `IAMAPIKey`. | `string` | `"TrustedProfile"` | no | +| [logs\_agent\_log\_source\_namespaces](#input\_logs\_agent\_log\_source\_namespaces) | The list of namespaces from which logs should be forwarded by agent. If namespaces are not listed, logs from all namespaces will be sent. | `list(string)` | `[]` | no | +| [logs\_agent\_name](#input\_logs\_agent\_name) | The name of the Logs agent. The name is used in all Kubernetes and Helm resources in the cluster. | `string` | `"logs-agent"` | no | +| [logs\_agent\_namespace](#input\_logs\_agent\_namespace) | The namespace where the Logs agent is deployed. The default value is `ibm-agent`. | `string` | `"ibm-agent"` | no | +| [logs\_agent\_resources](#input\_logs\_agent\_resources) | The resources configuration for cpu/memory/storage. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources) | 
object({
    limits = object({
      cpu    = string
      memory = string
    })
    requests = object({
      cpu    = string
      memory = string
    })
  })
| 
{
  "limits": {
    "cpu": "500m",
    "memory": "3Gi"
  },
  "requests": {
    "cpu": "100m",
    "memory": "1Gi"
  }
}
| no | +| [logs\_agent\_selected\_log\_source\_paths](#input\_logs\_agent\_selected\_log\_source\_paths) | The list of specific log sources paths. Logs will only be collected from the specified log source paths. If no paths are specified, it will send logs from `/var/log/containers`. | `list(string)` | `[]` | no | +| [logs\_agent\_tolerations](#input\_logs\_agent\_tolerations) | List of tolerations to apply to Logs agent. The default value means a pod will run on every node. | 
list(object({
    key               = optional(string)
    operator          = optional(string)
    value             = optional(string)
    effect            = optional(string)
    tolerationSeconds = optional(number)
  }))
| 
[
  {
    "operator": "Exists"
  }
]
| no | +| [logs\_agent\_trusted\_profile](#input\_logs\_agent\_trusted\_profile) | The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role. | `string` | `null` | no | +| [logs\_agent\_version](#input\_logs\_agent\_version) | The version of the Logs agent to deploy. | `string` | `"1.4.2"` | no | +| [wait\_till](#input\_wait\_till) | To avoid long wait times when you run your Terraform code, you can specify the stage when you want Terraform to mark the cluster resource creation as completed. Depending on what stage you choose, the cluster creation might not be fully completed and continues to run in the background. However, your Terraform code can continue to run without waiting for the cluster to be fully created. Supported args are `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady` and `Normal` | `string` | `"Normal"` | no | +| [wait\_till\_timeout](#input\_wait\_till\_timeout) | Timeout for wait\_till in minutes. | `number` | `90` | no | ### Outputs -| Name | Description | -|------|-------------| -| [account\_id](#output\_account\_id) | An alpha-numeric value identifying the account ID. | -| [crn](#output\_crn) | The CRN of the resource instance. | -| [guid](#output\_guid) | The GUID of the resource instance. | -| [id](#output\_id) | The unique identifier of the resource instance. | +No outputs. - - -## Contributing - -You can report issues and request features for this module in GitHub issues in the module repo. See [Report an issue or request a feature](https://github.com/terraform-ibm-modules/.github/blob/main/.github/SUPPORT.md). - -To set up your local development environment, see [Local development setup](https://terraform-ibm-modules.github.io/documentation/#/local-dev-setup) in the project documentation. diff --git a/cra-config.yaml b/cra-config.yaml index 9a4c7faa..d495e495 100644 --- a/cra-config.yaml +++ b/cra-config.yaml @@ -1,17 +1,6 @@ -# -# Developer tips: -# - CRA = Code Risk Analyzer (more info on CRA: https://cloud.ibm.com/docs/code-risk-analyzer-cli-plugin?topic=code-risk-analyzer-cli-plugin-cra-cli-plugin) -# - Multiple directories can be scanned by CRA. Ensure if there are any deployable architecture in the repository that they are all scanned -# - More info about supported configurations at https://github.com/terraform-ibm-modules/common-pipeline-assets/blob/main/.github/workflows/terraform-test-pipeline.md#cra-config-yaml -# - +# More info about this file at https://github.com/terraform-ibm-modules/common-pipeline-assets/blob/main/.github/workflows/terraform-test-pipeline.md#cra-config-yaml version: "v1" CRA_TARGETS: - - CRA_TARGET: "examples/advanced" # Target directory for CRA scan. If not provided, the CRA Scan will not be run. - CRA_IGNORE_RULES_FILE: "cra-tf-validate-ignore-rules.json" - PROFILE_ID: "fe96bd4d-9b37-40f2-b39f-a62760e326a3" # SCC profile ID (currently set to 'IBM Cloud Framework for Financial Services' '1.7.0' profile). - # SCC_INSTANCE_ID: "" # The SCC instance ID to use to download profile for CRA scan. If not provided, a default global value will be used. - # SCC_REGION: "" # The IBM Cloud region that the SCC instance is in. If not provided, a default global value will be used. - CRA_ENVIRONMENT_VARIABLES: # An optional map of environment variables for CRA, where the key is the variable name and value is the value. Useful for providing TF_VARs. - TF_VAR_prefix: "mock" - TF_VAR_region: "us-south" + - CRA_TARGET: "examples/logs-agent-ocp" # Target directory for CRA scan. If not provided, the CRA Scan will not be run. + CRA_IGNORE_RULES_FILE: "cra-tf-validate-ignore-rules.json" # CRA Ignore file to use. If not provided, it checks the repo root directory for `cra-tf-validate-ignore-rules.json` + PROFILE_ID: "fe96bd4d-9b37-40f2-b39f-a62760e326a3" # SCC profile ID (currently set to 'IBM Cloud Framework for Financial Services' '1.7.0' profile). diff --git a/cra-tf-validate-ignore-rules.json b/cra-tf-validate-ignore-rules.json index adbff6e0..45bef2ac 100644 --- a/cra-tf-validate-ignore-rules.json +++ b/cra-tf-validate-ignore-rules.json @@ -1,3 +1,23 @@ + { - "scc_rules": [] + "scc_rules": [ + { + "scc_rule_id": "rule-216e2449-27d7-4afc-929a-b66e196a9cf9", + "description": "Check whether Flow Logs for VPC are enabled", + "ignore_reason": "This rule is not relevant to the module itself, just the VPC resource that is used in the example that is scanned", + "is_valid": false + }, + { + "scc_rule_id": "rule-2325054a-c338-474a-9740-0b7034487e40", + "description:": "Check whether OpenShift clusters are accessible only by using private endpoints", + "ignore_reason": "This rule is not relevant to the module itself, just the cluster resource that is used in the example that is scanned", + "is_valid": false + }, + { + "scc_rule_id": "rule-64c0bea0-8760-4a6b-a56c-ee375a48961e", + "description:": "Check whether Virtual Private Cloud (VPC) has no public gateways attached", + "ignore_reason": "This rule is not relevant to the module itself, just the VPC resource that is used in the example that is scanned", + "is_valid": false + } + ] } diff --git a/examples/advanced/README.md b/examples/advanced/README.md deleted file mode 100644 index d52511a3..00000000 --- a/examples/advanced/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Advanced example - - - diff --git a/examples/advanced/main.tf b/examples/advanced/main.tf deleted file mode 100644 index 29e41045..00000000 --- a/examples/advanced/main.tf +++ /dev/null @@ -1,32 +0,0 @@ -######################################################################################################################## -# Resource group -######################################################################################################################## - -module "resource_group" { - source = "terraform-ibm-modules/resource-group/ibm" - version = "1.1.6" - # if an existing resource group is not set (null) create a new one using prefix - resource_group_name = var.resource_group == null ? "${var.prefix}-resource-group" : null - existing_resource_group_name = var.resource_group -} - -######################################################################################################################## -# COS -######################################################################################################################## - -# -# Developer tips: -# - Call the local module / modules in the example to show how they can be consumed -# - Include the actual module source as a code comment like below so consumers know how to consume from correct location -# - -module "cos" { - source = "../.." - # remove the above line and uncomment the below 2 lines to consume the module from the registry - # source = "terraform-ibm-modules//ibm" - # version = "X.Y.Z" # Replace "X.Y.Z" with a release version to lock into a specific release - name = "${var.prefix}-cos" - resource_group_id = module.resource_group.resource_group_id - resource_tags = var.resource_tags - plan = "cos-one-rate-plan" -} diff --git a/examples/advanced/outputs.tf b/examples/advanced/outputs.tf deleted file mode 100644 index 316751fb..00000000 --- a/examples/advanced/outputs.tf +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Outputs -############################################################################## - -# -# Developer tips: -# - Include all relevant outputs from the modules being called in the example -# - -output "account_id" { - description = "An alpha-numeric value identifying the account ID." - value = module.cos.account_id -} - -output "guid" { - description = "The GUID of the resource instance." - value = module.cos.account_id -} - -output "id" { - description = "The unique identifier of the resource instance." - value = module.cos.id -} - -output "crn" { - description = "The CRN of the resource instance." - value = module.cos.crn -} - -output "resource_group_name" { - description = "Resource group name." - value = module.resource_group.resource_group_name -} - -output "resource_group_id" { - description = "Resource group ID." - value = module.resource_group.resource_group_id -} diff --git a/examples/advanced/variables.tf b/examples/advanced/variables.tf deleted file mode 100644 index d4603642..00000000 --- a/examples/advanced/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -######################################################################################################################## -# Input variables -######################################################################################################################## - -# -# Module developer tips: -# - Examples are references that consumers can use to see how the module can be consumed. They are not designed to be -# flexible re-usable solutions for general consumption, so do not expose any more variables here and instead hard -# code things in the example main.tf with code comments explaining the different configurations. -# - For the same reason as above, do not add default values to the example inputs. -# - -variable "ibmcloud_api_key" { - type = string - description = "The IBM Cloud API Key." - sensitive = true -} - -variable "region" { - type = string - description = "Region to provision all resources created by this example." -} - -variable "prefix" { - type = string - description = "A string value to prefix to all resources created by this example." -} - -variable "resource_group" { - type = string - description = "The name of an existing resource group to provision resources in to. If not set a new resource group will be created using the prefix variable." - default = null -} - -variable "resource_tags" { - type = list(string) - description = "List of resource tag to associate with all resource instances created by this example." - default = [] -} diff --git a/examples/advanced/version.tf b/examples/advanced/version.tf deleted file mode 100644 index ecfa9780..00000000 --- a/examples/advanced/version.tf +++ /dev/null @@ -1,16 +0,0 @@ -terraform { - required_version = ">= 1.9.0" - - # - # Developer tips: - # - Ensure that there is always 1 example locked into the lowest provider version of the range defined in the main - # module's version.tf (usually a basic example), and 1 example that will always use the latest provider version. - # - - required_providers { - ibm = { - source = "IBM-Cloud/ibm" - version = ">= 1.71.2, < 2.0.0" - } - } -} diff --git a/examples/basic/README.md b/examples/basic/README.md deleted file mode 100644 index e5977ae2..00000000 --- a/examples/basic/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Basic example - - - -An end-to-end basic example that will provision the following: -- A new resource group if one is not passed in. -- A new standard plan Cloud Object Storage instance using the root level module. diff --git a/examples/basic/main.tf b/examples/basic/main.tf deleted file mode 100644 index 410054ad..00000000 --- a/examples/basic/main.tf +++ /dev/null @@ -1,31 +0,0 @@ -######################################################################################################################## -# Resource group -######################################################################################################################## - -module "resource_group" { - source = "terraform-ibm-modules/resource-group/ibm" - version = "1.1.6" - # if an existing resource group is not set (null) create a new one using prefix - resource_group_name = var.resource_group == null ? "${var.prefix}-resource-group" : null - existing_resource_group_name = var.resource_group -} - -######################################################################################################################## -# COS -######################################################################################################################## - -# -# Developer tips: -# - Call the local module / modules in the example to show how they can be consumed -# - include the actual module source as a code comment like below so consumers know how to consume from correct location -# - -module "cos" { - source = "../.." - # remove the above line and uncomment the below 2 lines to consume the module from the registry - # source = "terraform-ibm-modules//ibm" - # version = "X.Y.Z" # Replace "X.Y.Z" with a release version to lock into a specific release - name = "${var.prefix}-cos" - resource_group_id = module.resource_group.resource_group_id - resource_tags = var.resource_tags -} diff --git a/examples/basic/outputs.tf b/examples/basic/outputs.tf deleted file mode 100644 index 552db482..00000000 --- a/examples/basic/outputs.tf +++ /dev/null @@ -1,38 +0,0 @@ -######################################################################################################################## -# Outputs -######################################################################################################################## - -# -# Developer tips: -# - Include all relevant outputs from the modules being called in the example -# - -output "account_id" { - description = "An alpha-numeric value identifying the account ID." - value = module.cos.account_id -} - -output "guid" { - description = "The GUID of the resource instance." - value = module.cos.account_id -} - -output "id" { - description = "The unique identifier of the resource instance." - value = module.cos.id -} - -output "crn" { - description = "The CRN of the resource instance." - value = module.cos.crn -} - -output "resource_group_name" { - description = "Resource group name." - value = module.resource_group.resource_group_name -} - -output "resource_group_id" { - description = "Resource group ID." - value = module.resource_group.resource_group_id -} diff --git a/examples/basic/provider.tf b/examples/basic/provider.tf deleted file mode 100644 index 84b69850..00000000 --- a/examples/basic/provider.tf +++ /dev/null @@ -1,8 +0,0 @@ -######################################################################################################################## -# Provider config -######################################################################################################################## - -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = var.region -} diff --git a/examples/basic/variables.tf b/examples/basic/variables.tf deleted file mode 100644 index d4603642..00000000 --- a/examples/basic/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -######################################################################################################################## -# Input variables -######################################################################################################################## - -# -# Module developer tips: -# - Examples are references that consumers can use to see how the module can be consumed. They are not designed to be -# flexible re-usable solutions for general consumption, so do not expose any more variables here and instead hard -# code things in the example main.tf with code comments explaining the different configurations. -# - For the same reason as above, do not add default values to the example inputs. -# - -variable "ibmcloud_api_key" { - type = string - description = "The IBM Cloud API Key." - sensitive = true -} - -variable "region" { - type = string - description = "Region to provision all resources created by this example." -} - -variable "prefix" { - type = string - description = "A string value to prefix to all resources created by this example." -} - -variable "resource_group" { - type = string - description = "The name of an existing resource group to provision resources in to. If not set a new resource group will be created using the prefix variable." - default = null -} - -variable "resource_tags" { - type = list(string) - description = "List of resource tag to associate with all resource instances created by this example." - default = [] -} diff --git a/examples/basic/version.tf b/examples/basic/version.tf deleted file mode 100644 index 401504c5..00000000 --- a/examples/basic/version.tf +++ /dev/null @@ -1,16 +0,0 @@ -terraform { - required_version = ">= 1.9.0" - - # - # Developer tips: - # - Ensure that there is always 1 example locked into the lowest provider version of the range defined in the main - # module's version.tf (usually a basic example), and 1 example that will always use the latest provider version. - # - - required_providers { - ibm = { - source = "IBM-Cloud/ibm" - version = "1.71.2" - } - } -} diff --git a/examples/logs-agent-iks/README.md b/examples/logs-agent-iks/README.md new file mode 100644 index 00000000..b13f2e14 --- /dev/null +++ b/examples/logs-agent-iks/README.md @@ -0,0 +1,11 @@ +# Logs agent on Kubernetes using CSE ingress endpoint with an apikey + +An example that shows how to deploy Logs agent in a Kubernetes cluster to send Logs directly to IBM Cloud Logs instance. + +The example provisions the following resources: +- A new resource group, if an existing one is not passed in. +- A basic VPC (if `is_vpc_cluster` is true). +- A Kubernetes cluster. +- A Service ID with `Sender` role to `logs` service and an apikey. +- An IBM Cloud Logs instance. +- Logs agent. diff --git a/examples/logs-agent-iks/main.tf b/examples/logs-agent-iks/main.tf new file mode 100644 index 00000000..4653c380 --- /dev/null +++ b/examples/logs-agent-iks/main.tf @@ -0,0 +1,156 @@ +############################################################################## +# Resource Group +############################################################################## + +module "resource_group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.6" + # if an existing resource group is not set (null) create a new one using prefix + resource_group_name = var.resource_group == null ? "${var.prefix}-resource-group" : null + existing_resource_group_name = var.resource_group +} + +############################################################################## +# Service ID with logs sender role + apikey +############################################################################## + +# As a `Sender`, you can send logs to your IBM Cloud Logs service instance - but not query or tail logs. This role is meant to be used by agent and routers sending logs. +module "iam_service_id" { + source = "terraform-ibm-modules/iam-service-id/ibm" + version = "1.2.0" + iam_service_id_name = "${var.prefix}-service-id" + iam_service_id_description = "Logs Agent service id" + iam_service_id_apikey_provision = true + iam_service_policies = { + logs = { + roles = ["Sender"] + resources = [{ + service = "logs" + }] + } + } +} + +############################################################################## +# Create VPC and IKS Cluster +############################################################################## + +resource "ibm_is_vpc" "example_vpc" { + count = var.is_vpc_cluster ? 1 : 0 + name = "${var.prefix}-vpc" + resource_group = module.resource_group.resource_group_id + tags = var.resource_tags +} + +resource "ibm_is_subnet" "testacc_subnet" { + count = var.is_vpc_cluster ? 1 : 0 + name = "${var.prefix}-subnet" + vpc = ibm_is_vpc.example_vpc[0].id + zone = "${var.region}-1" + total_ipv4_address_count = 256 + resource_group = module.resource_group.resource_group_id +} + +# Lookup the current default kube version +data "ibm_container_cluster_versions" "cluster_versions" {} +locals { + default_version = data.ibm_container_cluster_versions.cluster_versions.default_kube_version +} + +resource "ibm_container_vpc_cluster" "cluster" { + count = var.is_vpc_cluster ? 1 : 0 + name = var.prefix + vpc_id = ibm_is_vpc.example_vpc[0].id + kube_version = local.default_version + flavor = "bx2.4x16" + worker_count = "2" + force_delete_storage = true + wait_till = "IngressReady" + zones { + subnet_id = ibm_is_subnet.testacc_subnet[0].id + name = "${var.region}-1" + } + resource_group_id = module.resource_group.resource_group_id + tags = var.resource_tags +} + +resource "ibm_container_cluster" "cluster" { + #checkov:skip=CKV2_IBM_7:Public endpoint is required for testing purposes + count = var.is_vpc_cluster ? 0 : 1 + name = var.prefix + datacenter = var.datacenter + default_pool_size = 2 + hardware = "shared" + kube_version = local.default_version + force_delete_storage = true + machine_type = "b3c.4x16" + public_vlan_id = ibm_network_vlan.public_vlan[0].id + private_vlan_id = ibm_network_vlan.private_vlan[0].id + wait_till = "Normal" + resource_group_id = module.resource_group.resource_group_id + tags = var.resource_tags + + timeouts { + delete = "2h" + create = "3h" + } +} + +locals { + cluster_name_id = var.is_vpc_cluster ? ibm_container_vpc_cluster.cluster[0].id : ibm_container_cluster.cluster[0].id +} + +resource "ibm_network_vlan" "public_vlan" { + count = var.is_vpc_cluster ? 0 : 1 + datacenter = var.datacenter + type = "PUBLIC" +} + +resource "ibm_network_vlan" "private_vlan" { + count = var.is_vpc_cluster ? 0 : 1 + datacenter = var.datacenter + type = "PRIVATE" +} + +data "ibm_container_cluster_config" "cluster_config" { + cluster_name_id = local.cluster_name_id + resource_group_id = module.resource_group.resource_group_id +} + +# Sleep to allow RBAC sync on cluster +resource "time_sleep" "wait_operators" { + depends_on = [data.ibm_container_cluster_config.cluster_config] + create_duration = "45s" +} + +############################################################################## +# Cloud Logs Instance +############################################################################## + +module "cloud_logs" { + source = "terraform-ibm-modules/cloud-logs/ibm" + version = "1.0.0" + resource_group_id = module.resource_group.resource_group_id + plan = "standard" + region = var.region + instance_name = "${var.prefix}-cloud-logs" + resource_tags = var.resource_tags +} + +############################################################################## +# Logs Agent +############################################################################## + +module "logs_agent" { + source = "../.." + depends_on = [time_sleep.wait_operators] + cluster_id = local.cluster_name_id + is_vpc_cluster = var.is_vpc_cluster + cluster_resource_group_id = module.resource_group.resource_group_id + # Logs Agent + logs_agent_iam_mode = "IAMAPIKey" + logs_agent_iam_api_key = module.iam_service_id.service_id_apikey + cloud_logs_ingress_endpoint = module.cloud_logs.ingress_private_endpoint + cloud_logs_ingress_port = 3443 + logs_agent_enable_scc = false # only true for Openshift +} diff --git a/examples/logs-agent-iks/outputs.tf b/examples/logs-agent-iks/outputs.tf new file mode 100644 index 00000000..135186b7 --- /dev/null +++ b/examples/logs-agent-iks/outputs.tf @@ -0,0 +1,11 @@ +############################################################################## +# Outputs +############################################################################## + +#output "myoutput" { +# description = "Description of my output" +# value = "value" +# depends_on = [] +#} + +############################################################################## diff --git a/examples/logs-agent-iks/provider.tf b/examples/logs-agent-iks/provider.tf new file mode 100644 index 00000000..76346099 --- /dev/null +++ b/examples/logs-agent-iks/provider.tf @@ -0,0 +1,24 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region +} + +provider "helm" { + kubernetes { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate + } + # IBM Cloud credentials are required to authenticate to the helm repo + registry { + url = "oci://icr.io/ibm/observe/logs-agent-helm" + username = "iamapikey" + password = var.ibmcloud_api_key + } +} + +provider "kubernetes" { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate +} diff --git a/examples/logs-agent-iks/variables.tf b/examples/logs-agent-iks/variables.tf new file mode 100644 index 00000000..f582f21a --- /dev/null +++ b/examples/logs-agent-iks/variables.tf @@ -0,0 +1,41 @@ +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud api token" + sensitive = true +} + +variable "prefix" { + type = string + description = "A prefix for the name of all resources that are created by this example" + default = "logs-agent-iks" +} + +variable "resource_group" { + type = string + description = "An existing resource group name to use for this example. If not specified, a new resource group is created." + default = null +} + +variable "resource_tags" { + type = list(string) + description = "A list of tags to add to the resources that are created." + default = [] +} + +variable "region" { + type = string + description = "The region where the resources are created." + default = "au-syd" +} + +variable "is_vpc_cluster" { + type = bool + description = "Specify true if the target cluster for the logs agent is a VPC cluster, false if it is classic cluster." + default = true +} + +variable "datacenter" { + type = string + description = "If creating a classic cluster, the data center where the cluster is created" + default = "syd01" +} diff --git a/examples/logs-agent-iks/version.tf b/examples/logs-agent-iks/version.tf new file mode 100644 index 00000000..8033b514 --- /dev/null +++ b/examples/logs-agent-iks/version.tf @@ -0,0 +1,26 @@ +terraform { + required_version = ">= 1.9.0" + + # Ensure that there is always 1 example locked into the lowest provider version of the range defined in the main + # module's version.tf (this example), and 1 example that will always use the latest provider version (logs-agent-ocp). + required_providers { + ibm = { + source = "ibm-cloud/ibm" + version = "1.76.1" + } + helm = { + source = "hashicorp/helm" + version = "2.15.0" + } + # The kubernetes provider is not actually required by the module itself, just this example, so OK to use ">=" here instead of locking into a version + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.16.1" + } + # The time provider is not actually required by the module itself, just this example, so OK to use ">=" here instead of locking into a version + time = { + source = "hashicorp/time" + version = ">= 0.9.1" + } + } +} diff --git a/examples/logs-agent-ocp/README.md b/examples/logs-agent-ocp/README.md new file mode 100644 index 00000000..88b76942 --- /dev/null +++ b/examples/logs-agent-ocp/README.md @@ -0,0 +1,13 @@ +# Logs agent on OCP using VPE ingress endpoint with a Trusted Profile + +An example that shows how to deploy Logs Routing agent in an Red Hat OpenShift container platform cluster to send Logs directly to IBM Cloud Logs instance respectively. + +The example provisions the following resources: + +- A new resource group, if an existing one is not passed in. +- A basic VPC. +- A Red Hat OpenShift Container Platform VPC cluster. +- A Trusted Profile with `Sender` role to `logs` service. +- An IBM Cloud Logs instance. +- A Virtual Private Endpoint for Cloud Logs. +- Logs agent. diff --git a/examples/logs-agent-ocp/main.tf b/examples/logs-agent-ocp/main.tf new file mode 100644 index 00000000..d3dd4d35 --- /dev/null +++ b/examples/logs-agent-ocp/main.tf @@ -0,0 +1,203 @@ +############################################################################## +# Resource Group +############################################################################## + +module "resource_group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.6" + # if an existing resource group is not set (null) create a new one using prefix + resource_group_name = var.resource_group == null ? "${var.prefix}-resource-group" : null + existing_resource_group_name = var.resource_group +} + +############################################################################## +# Trusted Profile +############################################################################## + +locals { + logs_agent_namespace = "ibm-agent" + logs_agent_name = "logs-agent" +} + + +module "trusted_profile" { + source = "terraform-ibm-modules/trusted-profile/ibm" + version = "1.0.5" + trusted_profile_name = "${var.prefix}-profile" + trusted_profile_description = "Logs agent Trusted Profile" + # As a `Sender`, you can send logs to your IBM Cloud Logs service instance - but not query or tail logs. This role is meant to be used by agent and routers sending logs. + trusted_profile_policies = [{ + roles = ["Sender"] + resources = [{ + service = "logs" + }] + }] + # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-agent` namespace. + trusted_profile_links = [{ + cr_type = "ROKS_SA" + links = [{ + crn = module.ocp_base.cluster_crn + namespace = local.logs_agent_namespace + name = local.logs_agent_name + }] + } + ] +} + +######################################################################################################################## +# VPC + Subnet + Public Gateway +# +# NOTE: This is a very simple VPC with single subnet in a single zone with a public gateway enabled, that will allow +# all traffic ingress/egress by default. +# For production use cases this would need to be enhanced by adding more subnets and zones for resiliency, and +# ACLs/Security Groups for network security. +######################################################################################################################## + +resource "ibm_is_vpc" "vpc" { + name = "${var.prefix}-vpc" + resource_group = module.resource_group.resource_group_id + address_prefix_management = "auto" + tags = var.resource_tags +} + +resource "ibm_is_public_gateway" "gateway" { + name = "${var.prefix}-gateway-1" + vpc = ibm_is_vpc.vpc.id + resource_group = module.resource_group.resource_group_id + zone = "${var.region}-1" +} + +resource "ibm_is_subnet" "subnet_zone_1" { + name = "${var.prefix}-subnet-1" + vpc = ibm_is_vpc.vpc.id + resource_group = module.resource_group.resource_group_id + zone = "${var.region}-1" + total_ipv4_address_count = 256 + public_gateway = ibm_is_public_gateway.gateway.id +} + +######################################################################################################################## +# OCP VPC cluster (single zone) +######################################################################################################################## + +locals { + cluster_vpc_subnets = { + default = [ + { + id = ibm_is_subnet.subnet_zone_1.id + cidr_block = ibm_is_subnet.subnet_zone_1.ipv4_cidr_block + zone = ibm_is_subnet.subnet_zone_1.zone + } + ] + } + + worker_pools = [ + { + subnet_prefix = "default" + pool_name = "default" # ibm_container_vpc_cluster automatically names default pool "default" (See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/2849) + machine_type = "bx2.4x16" + operating_system = "REDHAT_8_64" + workers_per_zone = 2 # minimum of 2 is allowed when using single zone + } + ] +} + +module "ocp_base" { + source = "terraform-ibm-modules/base-ocp-vpc/ibm" + version = "3.41.3" + resource_group_id = module.resource_group.resource_group_id + region = var.region + tags = var.resource_tags + cluster_name = var.prefix + force_delete_storage = true + vpc_id = ibm_is_vpc.vpc.id + vpc_subnets = local.cluster_vpc_subnets + ocp_version = var.ocp_version + worker_pools = local.worker_pools + access_tags = var.access_tags + ocp_entitlement = var.ocp_entitlement +} + +data "ibm_container_cluster_config" "cluster_config" { + cluster_name_id = module.ocp_base.cluster_id + resource_group_id = module.resource_group.resource_group_id +} + +############################################################################## +# Cloud Logs Instance +############################################################################## + +module "cloud_logs" { + source = "terraform-ibm-modules/cloud-logs/ibm" + version = "1.0.0" + resource_group_id = module.resource_group.resource_group_id + plan = "standard" + region = var.region + instance_name = "${var.prefix}-cloud-logs" + resource_tags = var.resource_tags +} + +data "ibm_is_security_groups" "vpc_security_groups" { + depends_on = [module.ocp_base] + vpc_id = ibm_is_vpc.vpc.id +} + +# The below code creates a VPE for Cloud logs in the provisioned VPC which allows the agent to access the private Cloud Logs Ingress endpoint. +module "vpe" { + source = "terraform-ibm-modules/vpe-gateway/ibm" + version = "4.3.0" + region = var.region + prefix = var.prefix + vpc_id = ibm_is_vpc.vpc.id + vpc_name = "${var.prefix}-vpc" + subnet_zone_list = [ + { + id = ibm_is_subnet.subnet_zone_1.id + name = ibm_is_subnet.subnet_zone_1.name + zone = ibm_is_subnet.subnet_zone_1.zone + } + ] + resource_group_id = module.resource_group.resource_group_id + security_group_ids = [for group in data.ibm_is_security_groups.vpc_security_groups.security_groups : group.id if group.name == "kube-${module.ocp_base.cluster_id}"] # Select only security group attached to the Cluster + cloud_service_by_crn = [ + { + crn = module.cloud_logs.crn + service_name = "logs" + } + ] + service_endpoints = "private" +} + +############################################################################## +# Logs Agent +############################################################################## + +module "logs_agent" { + source = "../.." + depends_on = [module.vpe] + cluster_id = module.ocp_base.cluster_id + cluster_resource_group_id = module.resource_group.resource_group_id + # Logs agent + logs_agent_trusted_profile = module.trusted_profile.trusted_profile.id + logs_agent_namespace = local.logs_agent_namespace + logs_agent_name = local.logs_agent_name + cloud_logs_ingress_endpoint = module.cloud_logs.ingress_private_endpoint + cloud_logs_ingress_port = 443 + # example of how to add additional metadata to the logs agent + logs_agent_additional_metadata = [{ + key = "cluster_id" + value = module.ocp_base.cluster_id + }] + logs_agent_resources = { + limits = { + cpu = "500m" + memory = "3Gi" + } + requests = { + cpu = "100m" + memory = "1Gi" + } + } + # example of how to add additional log source path + logs_agent_additional_log_source_paths = ["/logs/*.log"] +} diff --git a/examples/logs-agent-ocp/outputs.tf b/examples/logs-agent-ocp/outputs.tf new file mode 100644 index 00000000..2bb20333 --- /dev/null +++ b/examples/logs-agent-ocp/outputs.tf @@ -0,0 +1,12 @@ + +############################################################################## +# Outputs +############################################################################## + +#output "myoutput" { +# description = "Description of my output" +# value = "value" +# depends_on = [] +#} + +############################################################################## diff --git a/examples/logs-agent-ocp/provider.tf b/examples/logs-agent-ocp/provider.tf new file mode 100644 index 00000000..76346099 --- /dev/null +++ b/examples/logs-agent-ocp/provider.tf @@ -0,0 +1,24 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region +} + +provider "helm" { + kubernetes { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate + } + # IBM Cloud credentials are required to authenticate to the helm repo + registry { + url = "oci://icr.io/ibm/observe/logs-agent-helm" + username = "iamapikey" + password = var.ibmcloud_api_key + } +} + +provider "kubernetes" { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate +} diff --git a/examples/logs-agent-ocp/variables.tf b/examples/logs-agent-ocp/variables.tf new file mode 100644 index 00000000..2abf3487 --- /dev/null +++ b/examples/logs-agent-ocp/variables.tf @@ -0,0 +1,47 @@ +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud api token" + sensitive = true +} + +variable "prefix" { + type = string + description = "A prefix for the name of all resources that are created by this example" + default = "logs-agent-ocp" +} + +variable "resource_group" { + type = string + description = "An existing resource group name to use for this example. If not specified, a new resource group is created." + default = null +} + +variable "resource_tags" { + type = list(string) + description = "A list of tags to add to the resources that are created." + default = [] +} + +variable "access_tags" { + type = list(string) + description = "Optional list of access management tags to add to resources that are created" + default = [] +} + +variable "region" { + type = string + description = "The region where the resources are created." + default = "au-syd" +} + +variable "ocp_version" { + type = string + description = "Version of the OCP cluster to provision" + default = null +} + +variable "ocp_entitlement" { + type = string + description = "Value that is applied to the entitlements for OCP cluster provisioning" + default = null +} diff --git a/examples/logs-agent-ocp/version.tf b/examples/logs-agent-ocp/version.tf new file mode 100644 index 00000000..895b5a46 --- /dev/null +++ b/examples/logs-agent-ocp/version.tf @@ -0,0 +1,21 @@ + +terraform { + required_version = ">= 1.9.0" + + # Ensure that there is always 1 example locked into the lowest provider version of the range defined in the main + # module's version.tf (logs-agent-iks), and 1 example that will always use the latest provider version (this exammple). + required_providers { + ibm = { + source = "ibm-cloud/ibm" + version = "1.76.1" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.15.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.16.1" + } + } +} diff --git a/ibm_catalog.json b/ibm_catalog.json new file mode 100644 index 00000000..c8936bd6 --- /dev/null +++ b/ibm_catalog.json @@ -0,0 +1,215 @@ +{ + "products": [ + { + "label": "Cloud automation for Cloud Logs Agent", + "name": "deploy-arch-ibm-logs-agent", + "product_kind": "solution", + "tags": [ + "ibm_created", + "logging", + "logging_agent", + "terraform", + "target_terraform", + "solution" + ], + "keywords": [ + "logs", + "agent", + "IaC", + "infrastructure as code", + "terraform", + "solution" + ], + "provider_name": "IBM", + "short_description": "Deploys IBM Logs Agent to a cluster", + "long_description": "Solutions that support deploying IBM Logs Agent for logging.", + "offering_docs_url": "https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/README.md", + "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/main/images/observability-icon.svg", + "flavors": [ + { + "label": "Fully Configurable", + "name": "fully-configurable", + "working_directory": "solutions/fully-configurable", + "compliance": {}, + "iam_permissions": [ + { + "service_name": "containers-kubernetes", + "role_crns": [ + "crn:v1:bluemix:public:iam::::serviceRole:Manager", + "crn:v1:bluemix:public:iam::::role:Viewer" + ] + } + ], + "architecture": { + "features": [ + { + "title": "Works with any Openshift or k8s cluster", + "description": "Yes" + }, + { + "title": "Deploys the IBM Logs Agent on an existing cluster.", + "description": "Yes" + } + ], + "diagrams": [ + { + "diagram": { + "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/reference-architecture/deployable-architecture-logs-agent.svg", + "caption": "Logs Agent on a cluster", + "type": "image/svg+xml" + }, + "description": "Logs Agent deployed on a cluster." + } + ] + }, + "configuration": [ + { + "key": "ibmcloud_api_key", + "required": true + }, + { + "key": "cluster_id", + "custom_config": { + "type": "cluster_var", + "grouping": "deployment", + "original_grouping": "deployment" + } + }, + { + "key": "cluster_resource_group_id", + "custom_config": { + "type": "resource_group", + "grouping": "deployment", + "original_grouping": "deployment", + "config_constraints": { + "identifier": "rg_id" + } + } + }, + { + "key": "is_vpc_cluster", + "required": true + }, + { + "key": "is_ocp_cluster", + "required": true + }, + { + "key": "cloud_logs_ingress_endpoint", + "required": true + }, + { + "key": "cloud_logs_ingress_port", + "required": true + }, + { + "key": "logs_agent_trusted_profile_id", + "required": true + }, + { + "key": "logs_agent_chart_location" + }, + { + "key": "logs_agent_chart_version" + }, + { + "key": "logs_agent_version" + }, + { + "key": "logs_agent_resources" + }, + { + "key": "logs_agent_additional_log_source_paths" + }, + { + "key": "logs_agent_additional_metadata" + }, + { + "key": "logs_agent_exclude_log_source_paths" + }, + { + "key": "logs_agent_iam_api_key" + }, + { + "key": "logs_agent_iam_environment", + "options": [ + { + "displayname": "Production", + "value": "Production" + }, + { + "displayname": "Private Production", + "value": "PrivateProduction" + }, + { + "displayname": "Staging", + "value": "Staging" + }, + { + "displayname": "Private Staging", + "value": "PrivateStaging" + } + ] + }, + { + "key": "logs_agent_iam_mode", + "options": [ + { + "displayname": "Trusted Profile", + "value": "TrustedProfile" + }, + { + "displayname": "IAM API Key", + "value": "IAMAPIKey" + } + ] + }, + { + "key": "logs_agent_log_source_namespaces" + }, + { + "key": "logs_agent_name" + }, + { + "key": "logs_agent_namespace" + }, + { + "key": "logs_agent_selected_log_source_paths" + }, + { + "key": "logs_agent_tolerations" + }, + { + "key": "cluster_config_endpoint_type" + }, + { + "key": "wait_till", + "options": [ + { + "displayname": "Master Node Ready", + "value": "MasterNodeReady" + }, + { + "displayname": "OneWorkerNodeReady", + "value": "One Worker Node Ready" + }, + { + "displayname": "IngressReady", + "value": "Ingress Ready" + }, + { + "displayname": "Normal", + "value": "Normal" + } + ] + }, + { + "key": "wait_till_timeout" + } + ], + "install_type": "fullstack" + } + ] + } + ] +} diff --git a/images/logs-icon.svg b/images/logs-icon.svg new file mode 100644 index 00000000..77079220 --- /dev/null +++ b/images/logs-icon.svg @@ -0,0 +1 @@ + diff --git a/kubeconfig/.gitignore b/kubeconfig/.gitignore new file mode 100644 index 00000000..632a28fb --- /dev/null +++ b/kubeconfig/.gitignore @@ -0,0 +1,6 @@ +# Ignore everything +* + +# But not these files... +!.gitignore +!README.md diff --git a/kubeconfig/README.md b/kubeconfig/README.md new file mode 100644 index 00000000..dff9dd4a --- /dev/null +++ b/kubeconfig/README.md @@ -0,0 +1,2 @@ +This directory must exist in source control so the `ibm_container_cluster_config` data lookup can use it to place the +config.yml used to connect to a kubernetes cluster (See https://github.ibm.com/GoldenEye/issues/issues/552). diff --git a/main.tf b/main.tf index b6b879e2..92ee5a0c 100644 --- a/main.tf +++ b/main.tf @@ -1,13 +1,133 @@ -# -# Developer tips: -# - Below code should be replaced with the code for the root level module -# - -resource "ibm_resource_instance" "cos_instance" { - name = var.name - resource_group_id = var.resource_group_id - service = "cloud-object-storage" - plan = var.plan - location = "global" - tags = var.resource_tags +# Lookup cluster name from ID. The is_vpc_cluster variable defines whether to use the VPC data block or the Classic data block +data "ibm_container_vpc_cluster" "cluster" { + count = var.is_vpc_cluster ? 1 : 0 + name = var.cluster_id + resource_group_id = var.cluster_resource_group_id + wait_till = var.wait_till + wait_till_timeout = var.wait_till_timeout +} + +data "ibm_container_cluster" "cluster" { + count = var.is_vpc_cluster ? 0 : 1 + name = var.cluster_id + resource_group_id = var.cluster_resource_group_id + wait_till = var.wait_till + wait_till_timeout = var.wait_till_timeout +} + +# Download cluster config which is required to connect to cluster +data "ibm_container_cluster_config" "cluster_config" { + cluster_name_id = var.is_vpc_cluster ? data.ibm_container_vpc_cluster.cluster[0].name : data.ibm_container_cluster.cluster[0].name + resource_group_id = var.cluster_resource_group_id + config_dir = "${path.module}/kubeconfig" + endpoint_type = var.cluster_config_endpoint_type != "default" ? var.cluster_config_endpoint_type : null # null value represents default +} + +locals { + logs_agent_chart_location = var.logs_agent_chart_location + logs_agent_version = var.logs_agent_version != null ? var.logs_agent_version : var.logs_agent_chart_version # use logs_agent_chart_version somewhere so pre-commit passes until its needed + logs_agent_selected_log_source_paths = distinct(concat([for namespace in var.logs_agent_log_source_namespaces : "/var/log/containers/*_${namespace}_*.log"], var.logs_agent_selected_log_source_paths)) + logs_agent_iam_api_key = var.logs_agent_iam_api_key != null ? var.logs_agent_iam_api_key : "" + logs_agent_trusted_profile = var.logs_agent_trusted_profile != null ? var.logs_agent_trusted_profile : "" + cloud_logs_ingress_endpoint = var.cloud_logs_ingress_endpoint != null ? var.cloud_logs_ingress_endpoint : "" + logs_agent_additional_metadata = length(var.logs_agent_additional_metadata) > 0 ? merge([ + for metadata in var.logs_agent_additional_metadata : { + (metadata.key) = metadata.value + }]...) : {} # DO NOT REMOVE "...", it is used to convert list of objects into a single object + cluster_name = var.is_vpc_cluster ? data.ibm_container_vpc_cluster.cluster[0].resource_name : data.ibm_container_cluster.cluster[0].resource_name # Not publically documented in provider. See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/4485 +} + +resource "helm_release" "logs_agent" { + name = var.logs_agent_name + chart = local.logs_agent_chart_location + version = local.logs_agent_version + namespace = var.logs_agent_namespace + create_namespace = true + timeout = 1200 + wait = true + recreate_pods = true + force_update = true + + set { + name = "metadata.name" + type = "string" + value = var.logs_agent_name + } + set { + name = "image.version" + type = "string" + value = local.logs_agent_version + } + set { + name = "env.ingestionHost" + type = "string" + value = local.cloud_logs_ingress_endpoint + } + set { + name = "env.ingestionPort" + value = var.cloud_logs_ingress_port + } + set_sensitive { + name = "secret.iamAPIKey" + type = "string" + value = local.logs_agent_iam_api_key + } + set { + name = "env.trustedProfileID" + type = "string" + value = local.logs_agent_trusted_profile + } + set { + name = "env.iamMode" + type = "string" + value = var.logs_agent_iam_mode + } + set { + name = "env.iamEnvironment" + type = "string" + value = var.logs_agent_iam_environment + } + set { + name = "additionalLogSourcePaths" + type = "string" + value = join("\\,", var.logs_agent_additional_log_source_paths) + } + set { + name = "excludeLogSourcePaths" + type = "string" + value = join("\\,", var.logs_agent_exclude_log_source_paths) + } + set { + name = "selectedLogSourcePaths" + type = "string" + value = join("\\,", local.logs_agent_selected_log_source_paths) + } + set { + name = "clusterName" + type = "string" + value = local.cluster_name + } + set { + name = "scc.create" + value = var.logs_agent_enable_scc + } + + # dummy value hack to force update https://github.com/hashicorp/terraform-provider-helm/issues/515#issuecomment-813088122 + values = [ + yamlencode({ + tolerations = var.logs_agent_tolerations + resources = var.logs_agent_resources + additionalMetadata = local.logs_agent_additional_metadata + dummy = uuid() + }) + ] + + + provisioner "local-exec" { + command = "${path.module}/scripts/confirm-rollout-status.sh ${var.logs_agent_name} ${var.logs_agent_namespace}" + interpreter = ["/bin/bash", "-c"] + environment = { + KUBECONFIG = data.ibm_container_cluster_config.cluster_config.config_file_path + } + } } diff --git a/outputs.tf b/outputs.tf index 1c0cf4cc..135186b7 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,30 +1,11 @@ -######################################################################################################################## +############################################################################## # Outputs -######################################################################################################################## +############################################################################## -# -# Developer tips: -# - Below are some good practise sample outputs -# - They should be updated for outputs applicable to the module being added -# - Use variable validation when possible -# +#output "myoutput" { +# description = "Description of my output" +# value = "value" +# depends_on = [] +#} -output "account_id" { - description = "An alpha-numeric value identifying the account ID." - value = ibm_resource_instance.cos_instance.account_id -} - -output "guid" { - description = "The GUID of the resource instance." - value = ibm_resource_instance.cos_instance.account_id -} - -output "id" { - description = "The unique identifier of the resource instance." - value = ibm_resource_instance.cos_instance.id -} - -output "crn" { - description = "The CRN of the resource instance." - value = ibm_resource_instance.cos_instance.crn -} +############################################################################## diff --git a/reference-architecture/deployable-architecture-logs-agents.svg b/reference-architecture/deployable-architecture-logs-agents.svg new file mode 100644 index 00000000..3556cb0d --- /dev/null +++ b/reference-architecture/deployable-architecture-logs-agents.svg @@ -0,0 +1,4 @@ + + + +
IBM Cloud
IBM Cloud
Resource Group
Resource Group
Existing Cluster
Existing ClusterLogs Agent
data
dataCloud LogsText is not SVG - cannot display \ No newline at end of file diff --git a/renovate.json b/renovate.json index 8954b604..71db67b2 100644 --- a/renovate.json +++ b/renovate.json @@ -1,4 +1,15 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["github>terraform-ibm-modules/common-dev-assets:commonRenovateConfig"] + "extends": ["github>terraform-ibm-modules/common-dev-assets:commonRenovateConfig"], + "customManagers": [ + { + "customType": "regex", + "description": "Update agent version to the latest in variables.tf", + "fileMatch": ["variables.tf$"], + "datasourceTemplate": "docker", + "matchStrings": [ + "default\\s*=\\s*\"(?.*)\"\\s*# datasource: (?[^\\s]+)" + ] + } + ] } diff --git a/scripts/confirm-rollout-status.sh b/scripts/confirm-rollout-status.sh new file mode 100755 index 00000000..4197143d --- /dev/null +++ b/scripts/confirm-rollout-status.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +daemonset=$1 +namespace=$2 + +kubectl rollout status ds "${daemonset}" -n "${namespace}" --timeout 30m diff --git a/solutions/fully-configurable/DA-types.md b/solutions/fully-configurable/DA-types.md new file mode 100644 index 00000000..283ffb79 --- /dev/null +++ b/solutions/fully-configurable/DA-types.md @@ -0,0 +1,110 @@ +# Configuring Logs Agent Tolerations + +When you deploy the IBM Logs agent using the `terraform-ibm-logs-agent` module, you can configure the tolerations that the agent applies to its pods by using the `logs_agent_tolerations` variable. This variable allows you to specify tolerations for scheduling the logs agent pods on nodes with specific taints. + +### Options for `logs_agent_tolerations` +- `key` (optional): The taint key that the toleration applies to. +- `operator` (optional): The operator to use for the toleration. Valid values are `Exists` and `Equal`. +- `value` (optional): The value to match for the taint key. +- `effect` (optional): The effect of the taint to tolerate. Valid values are `NoSchedule`, `PreferNoSchedule`, and `NoExecute`. +- `tolerationSeconds` (optional): The duration (in seconds) for which the toleration is valid when the `effect` is `NoExecute`. + +### Example `logs_agent_tolerations` Usage + +To configure tolerations for the logs agent, you can set the `logs_agent_tolerations` variable as follows: + +```hcl +logs_agent_tolerations = [ + { + key = "example-key" + operator = "Equal" + value = "example-value" + effect = "NoSchedule" + }, + { + operator = "Exists" + } +] +``` + +In this example: +- The first toleration applies to nodes with a taint key of `example-key` and a value of `example-value`, with the `NoSchedule` effect. +- The second toleration applies to any taint key, regardless of value, with the `Exists` operator. + +### What It Does + +The `logs_agent_tolerations` variable is used to configure the tolerations for the logs agent pods. This allows the agent to run on nodes with specific taints. The configuration is passed to the Helm chart during deployment, ensuring that the logs agent pods are scheduled according to the specified tolerations. + +=================================================================== + +# Configuring Logs Agent Additional Metadata + +When you deploy the IBM logs agent using the `terraform-ibm-logs-agent` module, you can configure additional metadata fields to be added to the routed logs by using the `logs_agent_additional_metadata` variable. This variable allows you to specify key-value pairs of metadata that will be included in the logs. + +### Options for `logs_agent_additional_metadata` +- `key` (optional): The metadata key to add to the logs. +- `value` (optional): The metadata value to associate with the key. + +### Example `logs_agent_additional_metadata` Usage + +To configure additional metadata for the logs agent, you can set the `logs_agent_additional_metadata` variable as follows: + +```hcl +logs_agent_additional_metadata = [ + { + key = "environment" + value = "production" + }, + { + key = "team" + value = "devops" + } +] +``` + +In this example: +- The first metadata entry adds a key `environment` with the value `production` to the logs. +- The second metadata entry adds a key `team` with the value `devops` to the logs. + +### What It Does + +The `logs_agent_additional_metadata` variable is used to configure additional metadata fields that are added to the logs routed by the logs agent. This allows you to include custom metadata in the logs for better categorization or identification. The configuration is passed to the Helm chart during deployment, ensuring that the specified metadata is included in the logs. + +=================================================================== + +# Configuring Logs Agent Resources + +When you deploy the IBM Logs agent using the `terraform-ibm-logs-agent` module, you can configure the resource requests and limits for the logs agent pods by using the `logs_agent_resources` variable. This variable allows you to specify the CPU and memory resources allocated to the logs agent. + +### Options for `logs_agent_resources` +- `requests` (optional): Specifies the minimum amount of resources required. Includes: + - `cpu`: The amount of CPU requested (e.g., `100m` for 0.1 CPU). + - `memory`: The amount of memory requested (e.g., `128Mi` for 128 MiB). +- `limits` (optional): Specifies the maximum amount of resources allowed. Includes: + - `cpu`: The maximum amount of CPU allowed. + - `memory`: The maximum amount of memory allowed. + +### Example `logs_agent_resources` Usage + +To configure resource requests and limits for the logs agent, you can set the `logs_agent_resources` variable as follows: + +```hcl +logs_agent_resources = { + requests = { + cpu = "100m" + memory = "128Mi" + } + limits = { + cpu = "500m" + memory = "256Mi" + } +} +``` + +In this example: +- The `requests` section specifies that the logs agent requires at least `100m` CPU and `128Mi` memory to run. +- The `limits` section specifies that the logs agent can use up to `500m` CPU and `256Mi` memory. + +### What It Does + +The `logs_agent_resources` variable is used to configure the resource requests and limits for the logs agent pods. This ensures that the logs agent has sufficient resources to operate efficiently while preventing it from consuming excessive resources on the node. The configuration is passed to the Helm chart during deployment, ensuring that the specified resource constraints are applied. diff --git a/solutions/fully-configurable/README.md b/solutions/fully-configurable/README.md new file mode 100644 index 00000000..0b000b85 --- /dev/null +++ b/solutions/fully-configurable/README.md @@ -0,0 +1,11 @@ +# Cloud automation for Cloud Logs Agent + +This architecture deploys the following logs agent on a cluster: + +* Cloud Logs agent + +## Before you begin + +* Make sure that the Cluster is deployed. + +* Make sure that the Cloud Logs for which specific agent are required are deployed. diff --git a/solutions/fully-configurable/catalogValidationValues.json.template b/solutions/fully-configurable/catalogValidationValues.json.template new file mode 100644 index 00000000..f48a7e33 --- /dev/null +++ b/solutions/fully-configurable/catalogValidationValues.json.template @@ -0,0 +1,3 @@ +{ + "ibmcloud_api_key": $VALIDATION_APIKEY +} diff --git a/solutions/fully-configurable/kubeconfig/.gitignore b/solutions/fully-configurable/kubeconfig/.gitignore new file mode 100644 index 00000000..632a28fb --- /dev/null +++ b/solutions/fully-configurable/kubeconfig/.gitignore @@ -0,0 +1,6 @@ +# Ignore everything +* + +# But not these files... +!.gitignore +!README.md diff --git a/solutions/fully-configurable/kubeconfig/README.md b/solutions/fully-configurable/kubeconfig/README.md new file mode 100644 index 00000000..dff9dd4a --- /dev/null +++ b/solutions/fully-configurable/kubeconfig/README.md @@ -0,0 +1,2 @@ +This directory must exist in source control so the `ibm_container_cluster_config` data lookup can use it to place the +config.yml used to connect to a kubernetes cluster (See https://github.ibm.com/GoldenEye/issues/issues/552). diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf new file mode 100644 index 00000000..26518f13 --- /dev/null +++ b/solutions/fully-configurable/main.tf @@ -0,0 +1,45 @@ +############################################################################## +# Logs Agent +############################################################################## + +data "ibm_container_cluster_config" "cluster_config" { + cluster_name_id = local.is_vpc_cluster ? data.ibm_container_vpc_cluster.cluster[0].name : data.ibm_container_cluster.cluster[0].name + resource_group_id = var.cluster_resource_group_id + config_dir = "${path.module}/kubeconfig" + endpoint_type = local.cluster_config_endpoint_type != "default" ? local.cluster_config_endpoint_type : null +} + +locals { + cluster_config_endpoint_type = var.cluster_config_endpoint_type + is_vpc_cluster = var.is_vpc_cluster +} + +module "logs_agent" { + source = "../.." + cluster_id = var.cluster_id + cluster_resource_group_id = var.cluster_resource_group_id + cluster_config_endpoint_type = local.cluster_config_endpoint_type + # Logs Agent + logs_agent_chart_location = var.logs_agent_chart_location + logs_agent_chart_version = var.logs_agent_chart_version + logs_agent_version = var.logs_agent_version + logs_agent_name = var.logs_agent_name + logs_agent_namespace = var.logs_agent_namespace + logs_agent_trusted_profile = var.logs_agent_trusted_profile_id + logs_agent_iam_api_key = var.logs_agent_iam_api_key + logs_agent_tolerations = var.logs_agent_tolerations + logs_agent_additional_log_source_paths = var.logs_agent_additional_log_source_paths + logs_agent_exclude_log_source_paths = var.logs_agent_exclude_log_source_paths + logs_agent_selected_log_source_paths = var.logs_agent_selected_log_source_paths + logs_agent_log_source_namespaces = var.logs_agent_log_source_namespaces + logs_agent_iam_mode = var.logs_agent_iam_mode + logs_agent_iam_environment = var.logs_agent_iam_environment + logs_agent_additional_metadata = var.logs_agent_additional_metadata + logs_agent_enable_scc = var.is_ocp_cluster + logs_agent_resources = var.logs_agent_resources + cloud_logs_ingress_endpoint = var.cloud_logs_ingress_endpoint + cloud_logs_ingress_port = var.cloud_logs_ingress_port + is_vpc_cluster = var.is_vpc_cluster + wait_till = var.wait_till + wait_till_timeout = var.wait_till_timeout +} diff --git a/examples/advanced/provider.tf b/solutions/fully-configurable/outputs.tf similarity index 58% rename from examples/advanced/provider.tf rename to solutions/fully-configurable/outputs.tf index 2080946b..586e7dd2 100644 --- a/examples/advanced/provider.tf +++ b/solutions/fully-configurable/outputs.tf @@ -1,8 +1,6 @@ ############################################################################## -# Provider config +# Outputs ############################################################################## -provider "ibm" { - ibmcloud_api_key = var.ibmcloud_api_key - region = var.region -} + +############################################################################## diff --git a/solutions/fully-configurable/provider.tf b/solutions/fully-configurable/provider.tf new file mode 100644 index 00000000..63d2619c --- /dev/null +++ b/solutions/fully-configurable/provider.tf @@ -0,0 +1,37 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key +} + +provider "kubernetes" { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token +} + +provider "helm" { + kubernetes { + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + } + # IBM Cloud credentials are required to authenticate to the helm repo + registry { + url = "oci://icr.io/ibm/observe/logs-agent-helm" + username = "iamapikey" + password = var.ibmcloud_api_key + } +} + +# Retrieve information about an existing VPC cluster +data "ibm_container_vpc_cluster" "cluster" { + count = local.is_vpc_cluster ? 1 : 0 + name = var.cluster_id + wait_till = var.wait_till + wait_till_timeout = var.wait_till_timeout +} + +# Retrieve information about an existing Classic cluster +data "ibm_container_cluster" "cluster" { + count = local.is_vpc_cluster ? 0 : 1 + name = var.cluster_id + wait_till = var.wait_till + wait_till_timeout = var.wait_till_timeout +} diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf new file mode 100644 index 00000000..41eee762 --- /dev/null +++ b/solutions/fully-configurable/variables.tf @@ -0,0 +1,211 @@ +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud API key." + sensitive = true +} + +############################################################################## +# Cluster variables +############################################################################## + +variable "cluster_id" { + type = string + description = "The ID of the cluster to deploy the agent in." +} + +variable "cluster_resource_group_id" { + type = string + description = "The resource group ID of the cluster." +} + +variable "cluster_config_endpoint_type" { + description = "Specify the type of endpoint to use to access the cluster configuration. Possible values: `default`, `private`, `vpe`, `link`. The `default` value uses the default endpoint of the cluster." + type = string + default = "private" + nullable = false # use default if null is passed in +} + +variable "is_vpc_cluster" { + type = bool + description = "Specify true if the target cluster for the DA is a VPC cluster, false if it is classic cluster." + default = true +} + +variable "wait_till" { + description = "Specify the stage when Terraform should mark the cluster resource creation as completed. Supported values: `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady`, `Normal`." + type = string + default = "Normal" +} + +variable "wait_till_timeout" { + description = "Timeout for wait_till in minutes." + type = number + default = 90 +} + +############################################################################## +# Logs Agent variables +############################################################################## + +variable "logs_agent_version" { + description = "The version of the Logs agent to deploy." + type = string + default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false +} + +variable "logs_agent_chart_version" { + description = "The version of the helm chart to deploy." + type = string + default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false +} + +variable "logs_agent_chart_location" { + description = "The location of the Helm chart for the Logs agent." + type = string + default = "oci://icr.io/ibm/observe/logs-agent-helm" +} + +variable "logs_agent_name" { + description = "The name of the Logs agent. The name is used in all Kubernetes and Helm resources in the cluster." + type = string + default = "logs-agent" + nullable = false +} + +variable "logs_agent_namespace" { + type = string + description = "The namespace where the Logs agent is deployed. The default value is `ibm-agent`." + default = "ibm-agent" + nullable = false +} + +variable "logs_agent_trusted_profile_id" { + type = string + description = "The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role. Must provide a value for `logs_agent_iam_api_key` if `logs_agent_trusted_profile_id` is null." + default = null +} + +variable "logs_agent_iam_api_key" { + type = string + description = "The IBM Cloud API key for the Logs agent to authenticate and communicate with the IBM Cloud Logs. It is required if `logs_agent_iam_mode` is set to `IAMAPIKey`." + sensitive = true + default = null + validation { + condition = !(var.logs_agent_iam_mode == "IAMAPIKey" && var.logs_agent_iam_api_key == null) + error_message = "The `logs_agent_iam_api_key` is required when `logs_agent_iam_mode` is set to `IAMAPIKey`." + } +} + +variable "logs_agent_tolerations" { + description = "List of tolerations to apply to Logs agent. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md)." + type = list(object({ + key = optional(string) + operator = optional(string) + value = optional(string) + effect = optional(string) + tolerationSeconds = optional(number) + })) + default = [{ + operator = "Exists" + }] +} + +variable "logs_agent_resources" { + description = "The resources configuration for cpu/memory/storage. Learn more [here](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources) and [here](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md)." + type = object({ + limits = object({ + cpu = string + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + }) + default = { + limits = { + cpu = "500m" + memory = "3Gi" + } + requests = { + cpu = "100m" + memory = "1Gi" + } + } +} + +variable "logs_agent_additional_log_source_paths" { + type = list(string) + description = "The list of additional log sources. By default, the Logs agent collects logs from a single source at `/var/log/containers/logger-agent-ds-*.log`." + default = [] + nullable = false +} + +variable "logs_agent_exclude_log_source_paths" { + type = list(string) + description = "The list of log sources to exclude. Specify the paths that the Logs agent ignores." + default = [] + nullable = false +} + +variable "logs_agent_selected_log_source_paths" { + type = list(string) + description = "The list of specific log sources paths. Logs will only be collected from the specified log source paths." + default = [] + nullable = false +} + +variable "logs_agent_log_source_namespaces" { + type = list(string) + description = "The list of namespaces from which logs should be forwarded by agent. When specified logs from only these namespaces will be sent by the agent." + default = [] + nullable = false +} + +variable "logs_agent_iam_mode" { + type = string + default = "TrustedProfile" + description = "IAM authentication mode: `TrustedProfile` or `IAMAPIKey`." + validation { + error_message = "The IAM mode can only be `TrustedProfile` or `IAMAPIKey`." + condition = contains(["TrustedProfile", "IAMAPIKey"], var.logs_agent_iam_mode) + } +} + +variable "logs_agent_iam_environment" { + type = string + default = "PrivateProduction" + description = "IAM authentication Environment: `Production` or `PrivateProduction` or `Staging` or `PrivateStaging`." + validation { + error_message = "The IAM environment can only be `Production` or `PrivateProduction` or `Staging` or `PrivateStaging`." + condition = contains(["Production", "PrivateProduction", "Staging", "PrivateStaging"], var.logs_agent_iam_environment) + } +} + +variable "logs_agent_additional_metadata" { + description = "The list of additional metadata fields to add to the routed logs. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md)." + type = list(object({ + key = optional(string) + value = optional(string) + })) + default = [] +} + +variable "is_ocp_cluster" { + description = "Whether to enable creation of Security Context Constraints in Openshift. When installing on an OpenShift cluster, this setting is mandatory to configure permissions for pods within your cluster." + type = bool + default = true +} + +variable "cloud_logs_ingress_endpoint" { + description = "The host for IBM Cloud Logs ingestion. Ensure you use the ingress endpoint. See https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-endpoints_ingress." + type = string +} + +variable "cloud_logs_ingress_port" { + type = number + default = 3443 + description = "The target port for the IBM Cloud Logs ingestion endpoint. The port must be 443 if you connect by using a VPE gateway, or port 3443 when you connect by using CSEs." +} diff --git a/solutions/fully-configurable/version.tf b/solutions/fully-configurable/version.tf new file mode 100644 index 00000000..ef2dbfc9 --- /dev/null +++ b/solutions/fully-configurable/version.tf @@ -0,0 +1,20 @@ +terraform { + # module uses nullable feature which is only available in versions >= 1.1.0 + required_version = ">= 1.9.0" + + required_providers { + # Lock DA into an exact provider version - renovate automation will keep it updated + ibm = { + source = "ibm-cloud/ibm" + version = "1.76.1" + } + helm = { + source = "hashicorp/helm" + version = "2.16.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.33.0" + } + } +} diff --git a/tests/other_test.go b/tests/other_test.go index 88d360d3..4d00aa51 100644 --- a/tests/other_test.go +++ b/tests/other_test.go @@ -1,2 +1,23 @@ // Tests in this file are NOT run in the PR pipeline. They are run in the continuous testing pipeline along with the ones in pr_test.go package test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +const terraformDirLogsAgentROKS = "examples/logs-agent-ocp" + +func TestRunAgentVpcOcp(t *testing.T) { + t.Parallel() + + options := setupOptions(t, "logs-agent-roks", terraformDirLogsAgentROKS) + + // Keep costs down by using the cloud_pak entitlement + options.TerraformVars["ocp_entitlement"] = "cloud_pak" + + output, err := options.RunTestConsistency() + assert.Nil(t, err, "This should not have errored") + assert.NotNil(t, output, "Expected some output") +} diff --git a/tests/pr_test.go b/tests/pr_test.go index 8867ed00..93208fb7 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -1,60 +1,248 @@ -// Tests in this file are run in the PR pipeline and the continuous testing pipeline +// Tests in this file are run in the PR pipeline package test import ( + "fmt" + "os" + "strings" "testing" + "math/rand/v2" + + "github.com/gruntwork-io/terratest/modules/files" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/random" + "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/cloudinfo" "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testhelper" + "github.com/terraform-ibm-modules/ibmcloud-terratest-wrapper/testschematic" ) -// Use existing resource group -const resourceGroup = "geretain-test-resources" +const resourceGroup = "geretain-test-observability-agents" +const fullyConfigurableSolutionDir = "solutions/fully-configurable" +const fullyConfigurableSolutionKubeconfigDir = "solutions/fully-configurable/kubeconfig" +const terraformDirLogsAgentIKS = "examples/logs-agent-iks" + +var sharedInfoSvc *cloudinfo.CloudInfoService + +var validRegions = []string{ + "au-syd", + "eu-gb", + "eu-de", + "eu-es", + "us-south", +} + +// TestMain will be run before any parallel tests, used to set up a shared InfoService object to track region usage +// for multiple tests +func TestMain(m *testing.M) { + sharedInfoSvc, _ = cloudinfo.NewCloudInfoServiceFromEnv("TF_VAR_ibmcloud_api_key", cloudinfo.CloudInfoServiceOptions{}) + + os.Exit(m.Run()) +} -// Ensure every example directory has a corresponding test -const advancedExampleDir = "examples/advanced" -const basicExampleDir = "examples/basic" +func setupOptions(t *testing.T, prefix string, terraformDir string) *testhelper.TestOptions { -func setupOptions(t *testing.T, prefix string, dir string) *testhelper.TestOptions { options := testhelper.TestOptionsDefaultWithVars(&testhelper.TestOptions{ Testing: t, - TerraformDir: dir, + TerraformDir: terraformDir, Prefix: prefix, ResourceGroup: resourceGroup, + IgnoreUpdates: testhelper.Exemptions{ // Ignore for consistency check + List: []string{ + "module.logs_agent.helm_release.logs_agent", + }, + }, + CloudInfoService: sharedInfoSvc, }) + return options } -// Consistency test for the basic example -func TestRunBasicExample(t *testing.T) { +func TestFullyConfigurableSolution(t *testing.T) { t.Parallel() - options := setupOptions(t, "mod-template-basic", basicExampleDir) + var region = validRegions[rand.IntN(len(validRegions))] - output, err := options.RunTestConsistency() - assert.Nil(t, err, "This should not have errored") - assert.NotNil(t, output, "Expected some output") + // ------------------------------------------------------------------------------------------------------ + // Deploy SLZ ROKS Cluster and Observability instances since it is needed to deploy Logs Agent + // ------------------------------------------------------------------------------------------------------ + + prefix := fmt.Sprintf("slz-%s", strings.ToLower(random.UniqueId())) + realTerraformDir := "./resources" + tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId()))) + + // Verify ibmcloud_api_key variable is set + checkVariable := "TF_VAR_ibmcloud_api_key" + val, present := os.LookupEnv(checkVariable) + require.True(t, present, checkVariable+" environment variable not set") + require.NotEqual(t, "", val, checkVariable+" environment variable is empty") + + logger.Log(t, "Tempdir: ", tempTerraformDir) + existingTerraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: tempTerraformDir, + Vars: map[string]any{ + "prefix": prefix, + "region": region, + }, + // Set Upgrade to true to ensure latest version of providers and modules are used by terratest. + // This is the same as setting the -upgrade=true flag with terraform. + Upgrade: true, + }) + + terraform.WorkspaceSelectOrNew(t, existingTerraformOptions, prefix) + _, existErr := terraform.InitAndApplyE(t, existingTerraformOptions) + + if existErr != nil { + assert.True(t, existErr == nil, "Init and Apply of temp resources (SLZ-ROKS and Observability Instances) failed") + } else { + + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ + Testing: t, + Prefix: "logs-agent", + TarIncludePatterns: []string{ + "*.tf", + "kubeconfig/*.*", + "scripts/*.*", + fullyConfigurableSolutionDir + "/*.*", + fullyConfigurableSolutionKubeconfigDir + "/*.*", + }, + IgnoreUpdates: testhelper.Exemptions{ // Ignore for consistency check + List: []string{ + "module.logs_agent.helm_release.logs_agent", + }, + }, + ResourceGroup: resourceGroup, + TemplateFolder: fullyConfigurableSolutionDir, + Tags: []string{"test-schematic"}, + DeleteWorkspaceOnFail: false, + WaitJobCompleteMinutes: 60, + Region: region, + }) + + options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, + {Name: "cluster_id", Value: terraform.Output(t, existingTerraformOptions, "workload_cluster_id"), DataType: "string"}, + {Name: "logs_agent_trusted_profile_id", Value: terraform.Output(t, existingTerraformOptions, "trusted_profile_id"), DataType: "string"}, + {Name: "cloud_logs_ingress_endpoint", Value: terraform.Output(t, existingTerraformOptions, "cloud_logs_ingress_private_endpoint"), DataType: "string"}, + {Name: "cluster_resource_group_id", Value: terraform.Output(t, existingTerraformOptions, "cluster_resource_group_id"), DataType: "string"}, + } + + err := options.RunSchematicTest() + assert.Nil(t, err, "This should not have errored") + } + + // Check if "DO_NOT_DESTROY_ON_FAILURE" is set + envVal, _ := os.LookupEnv("DO_NOT_DESTROY_ON_FAILURE") + // Destroy the temporary existing resources if required + if t.Failed() && strings.ToLower(envVal) == "true" { + fmt.Println("Terratest failed. Debug the test and delete resources manually.") + } else { + logger.Log(t, "START: Destroy (existing resources)") + terraform.Destroy(t, existingTerraformOptions) + terraform.WorkspaceDelete(t, existingTerraformOptions, prefix) + logger.Log(t, "END: Destroy (existing resources)") + } } -func TestRunAdvancedExample(t *testing.T) { +func TestFullyConfigurableUpgradeSolution(t *testing.T) { t.Parallel() - options := setupOptions(t, "mod-template-adv", advancedExampleDir) + var region = validRegions[rand.IntN(len(validRegions))] + // ------------------------------------------------------------------------------------------------------ + // Deploy SLZ ROKS Cluster and Observability instances since it is needed to deploy Logs Agent + // ------------------------------------------------------------------------------------------------------ + + prefix := fmt.Sprintf("slz-%s", strings.ToLower(random.UniqueId())) + realTerraformDir := "./resources" + tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId()))) + + // Verify ibmcloud_api_key variable is set + checkVariable := "TF_VAR_ibmcloud_api_key" + val, present := os.LookupEnv(checkVariable) + require.True(t, present, checkVariable+" environment variable not set") + require.NotEqual(t, "", val, checkVariable+" environment variable is empty") + + logger.Log(t, "Tempdir: ", tempTerraformDir) + existingTerraformOptions := terraform.WithDefaultRetryableErrors(t, &terraform.Options{ + TerraformDir: tempTerraformDir, + Vars: map[string]any{ + "prefix": prefix, + "region": region, + }, + // Set Upgrade to true to ensure latest version of providers and modules are used by terratest. + // This is the same as setting the -upgrade=true flag with terraform. + Upgrade: true, + }) + + terraform.WorkspaceSelectOrNew(t, existingTerraformOptions, prefix) + _, existErr := terraform.InitAndApplyE(t, existingTerraformOptions) + + if existErr != nil { + assert.True(t, existErr == nil, "Init and Apply of temp resources (SLZ-ROKS and Observability Instances) failed") + } else { + + options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ + Testing: t, + Prefix: "logs-agent", + TarIncludePatterns: []string{ + "*.tf", + "kubeconfig/*.*", + "scripts/*.*", + fullyConfigurableSolutionDir + "/*.*", + fullyConfigurableSolutionKubeconfigDir + "/*.*", + }, + ResourceGroup: resourceGroup, + TemplateFolder: fullyConfigurableSolutionDir, + Tags: []string{"test-schematic"}, + DeleteWorkspaceOnFail: false, + WaitJobCompleteMinutes: 60, + Region: region, + }) + + options.TerraformVars = []testschematic.TestSchematicTerraformVar{ + {Name: "ibmcloud_api_key", Value: options.RequiredEnvironmentVars["TF_VAR_ibmcloud_api_key"], DataType: "string", Secure: true}, + {Name: "cluster_id", Value: terraform.Output(t, existingTerraformOptions, "workload_cluster_id"), DataType: "string"}, + {Name: "logs_agent_trusted_profile_id", Value: terraform.Output(t, existingTerraformOptions, "trusted_profile_id"), DataType: "string"}, + {Name: "cloud_logs_ingress_endpoint", Value: terraform.Output(t, existingTerraformOptions, "cloud_logs_ingress_private_endpoint"), DataType: "string"}, + {Name: "cluster_resource_group_id", Value: terraform.Output(t, existingTerraformOptions, "cluster_resource_group_id"), DataType: "string"}, + } + + err := options.RunSchematicUpgradeTest() + assert.Nil(t, err, "This should not have errored") + } + + // Check if "DO_NOT_DESTROY_ON_FAILURE" is set + envVal, _ := os.LookupEnv("DO_NOT_DESTROY_ON_FAILURE") + // Destroy the temporary existing resources if required + if t.Failed() && strings.ToLower(envVal) == "true" { + fmt.Println("Terratest failed. Debug the test and delete resources manually.") + } else { + logger.Log(t, "START: Destroy (existing resources)") + terraform.Destroy(t, existingTerraformOptions) + terraform.WorkspaceDelete(t, existingTerraformOptions, prefix) + logger.Log(t, "END: Destroy (existing resources)") + } +} + +func TestRunAgentClassicKubernetes(t *testing.T) { + t.Parallel() + + options := setupOptions(t, "logs-agent-iks", terraformDirLogsAgentIKS) + options.TerraformVars["is_vpc_cluster"] = false output, err := options.RunTestConsistency() assert.Nil(t, err, "This should not have errored") assert.NotNil(t, output, "Expected some output") } -// Upgrade test (using advanced example) -func TestRunUpgradeExample(t *testing.T) { +func TestRunAgentVpcKubernetes(t *testing.T) { t.Parallel() - options := setupOptions(t, "mod-template-adv-upg", advancedExampleDir) - - output, err := options.RunTestUpgrade() - if !options.UpgradeTestSkipped { - assert.Nil(t, err, "This should not have errored") - assert.NotNil(t, output, "Expected some output") - } + options := setupOptions(t, "logs-agent-iks", terraformDirLogsAgentIKS) + output, err := options.RunTestConsistency() + assert.Nil(t, err, "This should not have errored") + assert.NotNil(t, output, "Expected some output") } diff --git a/tests/resources/README.md b/tests/resources/README.md new file mode 100644 index 00000000..4bb3621d --- /dev/null +++ b/tests/resources/README.md @@ -0,0 +1 @@ +The terraform code in this directory is used by the existing resource test in tests/pr_test.go diff --git a/tests/resources/main.tf b/tests/resources/main.tf new file mode 100644 index 00000000..6297d354 --- /dev/null +++ b/tests/resources/main.tf @@ -0,0 +1,139 @@ +############################################################################## +# Resource Group +############################################################################## + +module "resource_group" { + source = "terraform-ibm-modules/resource-group/ibm" + version = "1.1.6" + # if an existing resource group is not set (null) create a new one using prefix + resource_group_name = var.resource_group == null ? "${var.prefix}-resource-group" : null + existing_resource_group_name = var.resource_group +} + +############################################################################## +# SLZ ROKS Pattern +############################################################################## + +module "landing_zone" { + source = "git::https://github.com/terraform-ibm-modules/terraform-ibm-landing-zone//patterns//roks//module?ref=v7.3.0" + region = var.region + prefix = var.prefix + tags = var.resource_tags + add_atracker_route = false + enable_transit_gateway = false + cluster_force_delete_storage = true + verify_cluster_network_readiness = false + use_ibm_cloud_private_api_endpoints = false + ignore_vpcs_for_cluster_deployment = ["management"] +} + +############################################################################## +# COS instance +############################################################################## + +module "cos" { + source = "terraform-ibm-modules/cos/ibm" + version = "8.16.4" + resource_group_id = module.resource_group.resource_group_id + region = var.region + cos_instance_name = "${var.prefix}-cos" + cos_tags = var.resource_tags + bucket_name = "${var.prefix}-bucket" + create_cos_bucket = false + retention_enabled = false # disable retention for test environments - enable for stage/prod + kms_encryption_enabled = false +} + + +############################################################################## +# COS buckets +############################################################################## + +locals { + logs_bucket_name = "${var.prefix}-logs-data" + metrics_bucket_name = "${var.prefix}-metrics-data" +} + +module "buckets" { + source = "terraform-ibm-modules/cos/ibm//modules/buckets" + version = "8.16.4" + bucket_configs = [ + { + bucket_name = local.logs_bucket_name + kms_encryption_enabled = false + region_location = var.region + resource_instance_id = module.cos.cos_instance_id + }, + { + bucket_name = local.metrics_bucket_name + kms_encryption_enabled = false + region_location = var.region + resource_instance_id = module.cos.cos_instance_id + } + ] +} + +############################################################################## +# Observability: +# - Cloud Logs instance +############################################################################## + +locals { + cluster_resource_group_id = module.landing_zone.cluster_data["${var.prefix}-workload-cluster"].resource_group_id + cluster_crn = module.landing_zone.cluster_data["${var.prefix}-workload-cluster"].crn +} + +module "cloud_logs" { + source = "terraform-ibm-modules/cloud-logs/ibm" + version = "1.0.0" + resource_group_id = local.cluster_resource_group_id + region = var.region + instance_name = "${var.prefix}-cloud-logs" + resource_tags = var.resource_tags + data_storage = { + # logs and metrics buckets must be different + logs_data = { + enabled = true + bucket_crn = module.buckets.buckets[local.logs_bucket_name].bucket_crn + bucket_endpoint = module.buckets.buckets[local.logs_bucket_name].s3_endpoint_direct + }, + metrics_data = { + enabled = true + bucket_crn = module.buckets.buckets[local.metrics_bucket_name].bucket_crn + bucket_endpoint = module.buckets.buckets[local.metrics_bucket_name].s3_endpoint_direct + } + } +} + +############################################################################## +# Trusted Profile +############################################################################## + +locals { + logs_agent_namespace = "ibm-agent" + logs_agent_name = "logs-agent" +} + +module "trusted_profile" { + source = "terraform-ibm-modules/trusted-profile/ibm" + version = "1.0.5" + trusted_profile_name = "${var.prefix}-profile" + trusted_profile_description = "Logs agent Trusted Profile" + # As a `Sender`, you can send logs to your IBM Cloud Logs service instance - but not query or tail logs. This role is meant to be used by agent and routers sending logs. + trusted_profile_policies = [{ + roles = ["Sender"] + resources = [{ + service = "logs" + }] + }] + # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-agent` namespace. + trusted_profile_links = [{ + cr_type = "ROKS_SA" + links = [{ + crn = local.cluster_crn + namespace = local.logs_agent_namespace + name = local.logs_agent_name + }] + } + ] +} diff --git a/tests/resources/outputs.tf b/tests/resources/outputs.tf new file mode 100644 index 00000000..bbc65ebb --- /dev/null +++ b/tests/resources/outputs.tf @@ -0,0 +1,48 @@ +############################################################################## +# Outputs +############################################################################## + +output "region" { + value = var.region + description = "Region where SLZ ROKS Cluster is deployed." +} + +output "prefix" { + value = module.landing_zone.prefix + description = "prefix" +} + +output "cluster_data" { + value = module.landing_zone.cluster_data + description = "Details of OCP cluster." +} + +output "workload_cluster_id" { + value = module.landing_zone.workload_cluster_id + description = "ID of the workload cluster." +} + +output "workload_cluster_crn" { + value = local.cluster_crn + description = "CRN of the workload cluster." +} + +output "cluster_resource_group_id" { + value = local.cluster_resource_group_id + description = "Resource group ID of the workload cluster." +} + +output "trusted_profile_id" { + value = module.trusted_profile.trusted_profile.id + description = "The ID of the trusted profile." +} + +output "cloud_logs_instance_name" { + value = module.cloud_logs.name + description = "The name of the provisioned IBM Cloud Logs instance." +} + +output "cloud_logs_ingress_private_endpoint" { + value = module.cloud_logs.ingress_private_endpoint + description = "The private ingress endpoint of the provisioned Cloud Logs instance." +} diff --git a/tests/resources/provider.tf b/tests/resources/provider.tf new file mode 100644 index 00000000..df45ef50 --- /dev/null +++ b/tests/resources/provider.tf @@ -0,0 +1,4 @@ +provider "ibm" { + ibmcloud_api_key = var.ibmcloud_api_key + region = var.region +} diff --git a/tests/resources/variables.tf b/tests/resources/variables.tf new file mode 100644 index 00000000..07a76de2 --- /dev/null +++ b/tests/resources/variables.tf @@ -0,0 +1,29 @@ +variable "ibmcloud_api_key" { + type = string + description = "The IBM Cloud API Key." + sensitive = true +} + +variable "region" { + type = string + description = "Region to provision all resources created by this example." + default = "us-south" +} + +variable "prefix" { + type = string + description = "Prefix to append to all resources created by this example." + default = "agent-da" +} + +variable "resource_tags" { + type = list(string) + description = "Optional list of tags to be added to created resources." + default = ["logs-agent-ocp"] +} + +variable "resource_group" { + type = string + description = "The name of an existing resource group to provision resources in. If not specified, a new resource group is created with the `prefix` variable." + default = null +} diff --git a/tests/resources/version.tf b/tests/resources/version.tf new file mode 100644 index 00000000..2ac74bab --- /dev/null +++ b/tests/resources/version.tf @@ -0,0 +1,10 @@ + +terraform { + required_version = ">= 1.9.0" + required_providers { + ibm = { + source = "ibm-cloud/ibm" + version = ">= 1.76.1" + } + } +} diff --git a/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh new file mode 100644 index 00000000..ba0f6092 --- /dev/null +++ b/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh @@ -0,0 +1,20 @@ +#! /bin/bash + +######################################################################################################################## +## This script is used by the catalog pipeline to destroy the SLZ OCP Cluster, which was provisioned as a ## +## prerequisite for the WAS extension that is published to the catalog ## +######################################################################################################################## + +set -e + +TERRAFORM_SOURCE_DIR="tests/resources" +TF_VARS_FILE="terraform.tfvars" + +( + cd ${TERRAFORM_SOURCE_DIR} + echo "Destroying prerequisite SLZ OCP Cluster and Observability instances .." + terraform destroy -input=false -auto-approve -var-file=${TF_VARS_FILE} || exit 1 + rm -f "${TF_VARS_FILE}" + + echo "Post-validation completed successfully" +) diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh new file mode 100644 index 00000000..b212d7bb --- /dev/null +++ b/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh @@ -0,0 +1,55 @@ +#! /bin/bash + +############################################################################################################ +## This script is used by the catalog pipeline to deploy the SLZ ROKS and Observability instances, +## which are the prerequisites for the Logs Agent extension. +############################################################################################################ + +set -e + +DA_DIR="solutions/fully-configurable" +TERRAFORM_SOURCE_DIR="tests/resources" +JSON_FILE="${DA_DIR}/catalogValidationValues.json" +REGION="us-south" +TF_VARS_FILE="terraform.tfvars" + +( + cwd=$(pwd) + cd ${TERRAFORM_SOURCE_DIR} + echo "Provisioning prerequisite SLZ ROKS CLUSTER and Observability Instances .." + terraform init || exit 1 + # $VALIDATION_APIKEY is available in the catalog runtime + { + echo "ibmcloud_api_key=\"${VALIDATION_APIKEY}\"" + echo "region=\"${REGION}\"" + echo "prefix=\"slz-$(openssl rand -hex 2)\"" + } >> ${TF_VARS_FILE} + terraform apply -input=false -auto-approve -var-file=${TF_VARS_FILE} || exit 1 + + region_var_name="region" + cluster_id_var_name="cluster_id" + cluster_id_value=$(terraform output -state=terraform.tfstate -raw workload_cluster_id) + cluster_resource_group_id_var_name="cluster_resource_group_id" + cluster_resource_group_id_value=$(terraform output -state=terraform.tfstate -raw cluster_resource_group_id) + logs_agent_trusted_profile_var_name="logs_agent_trusted_profile" + logs_agent_trusted_profile_value=$(terraform output -state=terraform.tfstate -raw trusted_profile_id) + cloud_logs_ingress_endpoint_var_name="cloud_logs_ingress_endpoint" + cloud_logs_ingress_endpoint_value=$(terraform output -state=terraform.tfstate -raw cloud_logs_ingress_private_endpoint) + + echo "Appending '${cluster_id_var_name}' and '${region_var_name}' input variable values to ${JSON_FILE}.." + + cd "${cwd}" + jq -r --arg region_var_name "${region_var_name}" \ + --arg region_var_value "${REGION}" \ + --arg cluster_id_var_name "${cluster_id_var_name}" \ + --arg cluster_id_value "${cluster_id_value}" \ + --arg cluster_resource_group_id_var_name "${cluster_resource_group_id_var_name}" \ + --arg cluster_resource_group_id_value "${cluster_resource_group_id_value}" \ + --arg logs_agent_trusted_profile_var_name "${logs_agent_trusted_profile_var_name}" \ + --arg logs_agent_trusted_profile_value "${logs_agent_trusted_profile_value}" \ + --arg cloud_logs_ingress_endpoint_var_name "${cloud_logs_ingress_endpoint_var_name}" \ + --arg cloud_logs_ingress_endpoint_value "${cloud_logs_ingress_endpoint_value}" \ + '. + {($region_var_name): $region_var_value, ($cluster_id_var_name): $cluster_id_value, ($cluster_resource_group_id_var_name): $cluster_resource_group_id_value, ($logs_agent_trusted_profile_var_name): $logs_agent_trusted_profile_value, ($cloud_logs_ingress_endpoint_var_name): $cloud_logs_ingress_endpoint_value}' "${JSON_FILE}" > tmpfile && mv tmpfile "${JSON_FILE}" || exit 1 + + echo "Pre-validation complete successfully" +) diff --git a/variables.tf b/variables.tf index a9d9899a..5d2a8e6f 100644 --- a/variables.tf +++ b/variables.tf @@ -1,36 +1,229 @@ -######################################################################################################################## -# Input Variables -######################################################################################################################## +############################################################################## +# Cluster variables +############################################################################## -# -# Developer tips: -# - Below are some common module input variables -# - They should be updated for input variables applicable to the module being added -# - Use variable validation when possible -# +variable "cluster_id" { + type = string + description = "The ID of the cluster to deploy the agent." +} + +variable "cluster_resource_group_id" { + type = string + description = "The resource group ID of the cluster." +} -variable "name" { +variable "cluster_config_endpoint_type" { + description = "The type of endpoint to use for the cluster config access: `default`, `private`, `vpe`, or `link`. The `default` value uses the default endpoint of the cluster." type = string - description = "A descriptive name used to identify the resource instance." + default = "default" + nullable = false # use default if null is passed in + validation { + error_message = "Invalid endpoint type. Valid values are `default`, `private`, `vpe`, or `link`." + condition = contains(["default", "private", "vpe", "link"], var.cluster_config_endpoint_type) + } } -variable "plan" { +variable "is_vpc_cluster" { + description = "Specify true if the target cluster for the agent is a VPC cluster, false if it is a classic cluster." + type = bool + default = true +} + +variable "wait_till" { + description = "To avoid long wait times when you run your Terraform code, you can specify the stage when you want Terraform to mark the cluster resource creation as completed. Depending on what stage you choose, the cluster creation might not be fully completed and continues to run in the background. However, your Terraform code can continue to run without waiting for the cluster to be fully created. Supported args are `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady` and `Normal`" type = string - description = "The name of the plan type supported by service." - default = "standard" + default = "Normal" + validation { - condition = contains(["standard", "cos-one-rate-plan"], var.plan) - error_message = "The specified pricing plan is not available. The following plans are supported: 'standard', 'cos-one-rate-plan'" + error_message = "`wait_till` value must be one of `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady` or `Normal`." + condition = contains([ + "MasterNodeReady", + "OneWorkerNodeReady", + "IngressReady", + "Normal" + ], var.wait_till) } } -variable "resource_group_id" { +variable "wait_till_timeout" { + description = "Timeout for wait_till in minutes." + type = number + default = 90 +} + +############################################################################## +# Logs Agent variables +############################################################################## + +variable "logs_agent_version" { + description = "The version of the Logs agent to deploy." + type = string + default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false +} + +variable "logs_agent_chart_version" { + description = "The version of the helm chart to deploy." + type = string + default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false +} + +variable "logs_agent_chart_location" { + description = "The location of the Helm chart for the Logs agent." type = string - description = "The ID of the resource group where you want to create the service." + default = "oci://icr.io/ibm/observe/logs-agent-helm" + nullable = false +} + +variable "logs_agent_name" { + description = "The name of the Logs agent. The name is used in all Kubernetes and Helm resources in the cluster." + type = string + default = "logs-agent" + nullable = false +} + +variable "logs_agent_namespace" { + type = string + description = "The namespace where the Logs agent is deployed. The default value is `ibm-agent`." + default = "ibm-agent" + nullable = false +} + +variable "logs_agent_trusted_profile" { + type = string + description = "The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role." + default = null + validation { + condition = !(var.logs_agent_trusted_profile == null && var.logs_agent_iam_mode == "TrustedProfile") + error_message = "The `logs_agent_trusted_profile` is required when `logs_agent_iam_mode` is set to `TrustedProfile`." + } +} + + +variable "logs_agent_iam_api_key" { + type = string + description = "The IBM Cloud API key for the Logs agent to authenticate and communicate with the IBM Cloud Logs. It is required if `logs_agent_iam_mode` is set to `IAMAPIKey`." + sensitive = true + default = null + validation { + condition = !(var.logs_agent_iam_mode == "IAMAPIKey" && var.logs_agent_iam_api_key == null) + error_message = "The `logs_agent_iam_api_key` is required when `logs_agent_iam_mode` is set to `IAMAPIKey`." + } +} + +variable "logs_agent_tolerations" { + description = "List of tolerations to apply to Logs agent. The default value means a pod will run on every node." + type = list(object({ + key = optional(string) + operator = optional(string) + value = optional(string) + effect = optional(string) + tolerationSeconds = optional(number) + })) + default = [{ + operator = "Exists" + }] +} + +variable "logs_agent_resources" { + description = "The resources configuration for cpu/memory/storage. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources)" + type = object({ + limits = object({ + cpu = string + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + }) + default = { + limits = { + cpu = "500m" + memory = "3Gi" + } + requests = { + cpu = "100m" + memory = "1Gi" + } + } +} + +variable "logs_agent_additional_log_source_paths" { + type = list(string) + description = "The list of additional log sources. By default, the Logs agent collects logs from a single source at `/var/log/containers/*.log`." + default = [] + nullable = false } -variable "resource_tags" { +variable "logs_agent_exclude_log_source_paths" { type = list(string) - description = "List of resource tag to associate with the instance." + description = "The list of log sources to exclude. Specify the paths that the Logs agent ignores." default = [] + nullable = false +} + +variable "logs_agent_selected_log_source_paths" { + type = list(string) + description = "The list of specific log sources paths. Logs will only be collected from the specified log source paths. If no paths are specified, it will send logs from `/var/log/containers`." + default = [] + nullable = false +} + +variable "logs_agent_log_source_namespaces" { + type = list(string) + description = "The list of namespaces from which logs should be forwarded by agent. If namespaces are not listed, logs from all namespaces will be sent." + default = [] + nullable = false +} + +variable "logs_agent_iam_mode" { + type = string + default = "TrustedProfile" + description = "IAM authentication mode: `TrustedProfile` or `IAMAPIKey`." + validation { + error_message = "The IAM mode can only be `TrustedProfile` or `IAMAPIKey`." + condition = contains(["TrustedProfile", "IAMAPIKey"], var.logs_agent_iam_mode) + } +} + +variable "logs_agent_iam_environment" { + type = string + default = "PrivateProduction" + description = "IAM authentication Environment: `Production` or `PrivateProduction` or `Staging` or `PrivateStaging`. `Production` specifies the public endpoint & `PrivateProduction` specifies the private endpoint." + validation { + error_message = "The IAM environment can only be `Production` or `PrivateProduction` or `Staging` or `PrivateStaging`." + condition = contains(["Production", "PrivateProduction", "Staging", "PrivateStaging"], var.logs_agent_iam_environment) + } +} + +variable "logs_agent_additional_metadata" { + description = "The list of additional metadata fields to add to the routed logs." + type = list(object({ + key = optional(string) + value = optional(string) + })) + default = [] +} + +variable "logs_agent_enable_scc" { + description = "Whether to enable creation of Security Context Constraints in Openshift. When installing on an OpenShift cluster, this setting is mandatory to configure permissions for pods within your cluster." + type = bool + default = true +} + +variable "cloud_logs_ingress_endpoint" { + description = "The host for IBM Cloud Logs ingestion. Ensure you use the ingress endpoint. See https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-endpoints_ingress." + type = string +} + +variable "cloud_logs_ingress_port" { + type = number + default = 3443 + description = "The target port for the IBM Cloud Logs ingestion endpoint. The port must be 443 if you connect by using a VPE gateway, or port 3443 when you connect by using CSEs." + validation { + error_message = "The Logs Routing supertenant ingestion port can only be `3443` or `443`." + condition = contains([3443, 443], var.cloud_logs_ingress_port) + } } diff --git a/version.tf b/version.tf index e51de7f6..a63e930b 100644 --- a/version.tf +++ b/version.tf @@ -1,18 +1,15 @@ terraform { - # require 1.9 or later to make use of cross-object referencing for input variable validations - # more info: https://www.hashicorp.com/blog/terraform-1-9-enhances-input-variable-validations required_version = ">= 1.9.0" - # - # Developer tips: - # - If your module requires any terraform providers, add them the "required_providers" section below. - # - Each required provider's version should be a flexible range to future proof the module's usage with upcoming minor and patch versions. - # - + # Each required provider's version should be a flexible range to future proof the module's usage with upcoming minor and patch versions. required_providers { ibm = { - source = "IBM-Cloud/ibm" - version = ">= 1.71.2, < 2.0.0" + source = "ibm-cloud/ibm" + version = ">= 1.76.1, <2.0.0" + } + helm = { + source = "hashicorp/helm" + version = ">= 2.15.0, <3.0.0" } } } From 0371c6c64cc04b2d966c1290c55e1d0028abf999 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 2 Apr 2025 16:31:11 +0100 Subject: [PATCH 02/21] fix: catalog.json SKIP UPGRADE TESTS --- ibm_catalog.json | 2 +- reference-architecture/deployable-architecture-logs-agent.svg | 4 ++++ .../deployable-architecture-logs-agents.svg | 4 ---- 3 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 reference-architecture/deployable-architecture-logs-agent.svg delete mode 100644 reference-architecture/deployable-architecture-logs-agents.svg diff --git a/ibm_catalog.json b/ibm_catalog.json index c8936bd6..ebf4adf0 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -24,7 +24,7 @@ "short_description": "Deploys IBM Logs Agent to a cluster", "long_description": "Solutions that support deploying IBM Logs Agent for logging.", "offering_docs_url": "https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/README.md", - "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/main/images/observability-icon.svg", + "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/main/images/logs-icon.svg", "flavors": [ { "label": "Fully Configurable", diff --git a/reference-architecture/deployable-architecture-logs-agent.svg b/reference-architecture/deployable-architecture-logs-agent.svg new file mode 100644 index 00000000..be4306fe --- /dev/null +++ b/reference-architecture/deployable-architecture-logs-agent.svg @@ -0,0 +1,4 @@ + + + +
IBM Cloud
IBM Cloud
Resource Group
Resource Group
Existing Cluster
Existing ClusterLogs Agent
data
dataCloud Logs InstanceText is not SVG - cannot display \ No newline at end of file diff --git a/reference-architecture/deployable-architecture-logs-agents.svg b/reference-architecture/deployable-architecture-logs-agents.svg deleted file mode 100644 index 3556cb0d..00000000 --- a/reference-architecture/deployable-architecture-logs-agents.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - -
IBM Cloud
IBM Cloud
Resource Group
Resource Group
Existing Cluster
Existing ClusterLogs Agent
data
dataCloud LogsText is not SVG - cannot display \ No newline at end of file From 31df561ba8ead90e069c525afaef68ac67823234 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 7 Apr 2025 20:42:05 +0100 Subject: [PATCH 03/21] fix: update code --- .github/settings.yml | 2 +- ibm_catalog.json | 32 +++++++++++++++---- .../deployable-architecture-logs-agent.svg | 2 +- solutions/fully-configurable/README.md | 12 ++++--- solutions/fully-configurable/provider.tf | 10 +++--- 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/.github/settings.yml b/.github/settings.yml index f0f5f18b..927feecc 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -22,7 +22,7 @@ repository: # Uncomment this description property # and update the description to the current repo description. - # description: "" + description: "Terraform module to deploy the IBM Cloud Logs Agent (LogDNA) on a Cluster" # Use a comma-separated list of topics to set on the repo (ensure not to use any caps in the topic string). topics: terraform, ibm-cloud, terraform-module, core-team, logs-agent, observability, logs, agent, cloud-logs, logging-agent diff --git a/ibm_catalog.json b/ibm_catalog.json index ebf4adf0..07622214 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -1,13 +1,13 @@ { "products": [ { - "label": "Cloud automation for Cloud Logs Agent", + "label": "Cloud automation for Logs Agent", "name": "deploy-arch-ibm-logs-agent", "product_kind": "solution", "tags": [ "ibm_created", "logging", - "logging_agent", + "logging_monitoring", "terraform", "target_terraform", "solution" @@ -180,7 +180,25 @@ "key": "logs_agent_tolerations" }, { - "key": "cluster_config_endpoint_type" + "key": "cluster_config_endpoint_type", + "options": [ + { + "displayname": "Default", + "value": "default" + }, + { + "displayname": "Private", + "value": "private" + }, + { + "displayname": "VPE", + "value": "vpe" + }, + { + "displayname": "Link", + "value": "link" + } + ] }, { "key": "wait_till", @@ -190,12 +208,12 @@ "value": "MasterNodeReady" }, { - "displayname": "OneWorkerNodeReady", - "value": "One Worker Node Ready" + "displayname": "One Worker Node Ready", + "value": "OneWorkerNodeReady" }, { - "displayname": "IngressReady", - "value": "Ingress Ready" + "displayname": "Ingress Ready", + "value": "IngressReady" }, { "displayname": "Normal", diff --git a/reference-architecture/deployable-architecture-logs-agent.svg b/reference-architecture/deployable-architecture-logs-agent.svg index be4306fe..a18343ac 100644 --- a/reference-architecture/deployable-architecture-logs-agent.svg +++ b/reference-architecture/deployable-architecture-logs-agent.svg @@ -1,4 +1,4 @@ -
IBM Cloud
IBM Cloud
Resource Group
Resource Group
Existing Cluster
Existing ClusterLogs Agent
data
dataCloud Logs InstanceText is not SVG - cannot display \ No newline at end of file +
IBM Cloud
IBM Cloud
Existing Resource Group
Existing Resource Group
Existing Cluster
Existing ClusterLogs Agent
metrics
metricsExisting Cloud LogsText is not SVG - cannot display \ No newline at end of file diff --git a/solutions/fully-configurable/README.md b/solutions/fully-configurable/README.md index 0b000b85..6f19f3c6 100644 --- a/solutions/fully-configurable/README.md +++ b/solutions/fully-configurable/README.md @@ -1,11 +1,15 @@ -# Cloud automation for Cloud Logs Agent +# Cloud automation for Logs Agent -This architecture deploys the following logs agent on a cluster: +This architecture deploys the following monitoring agent on a Red Hat OpenShift cluster: -* Cloud Logs agent +* Cloud Monitoring agent ## Before you begin * Make sure that the Cluster is deployed. -* Make sure that the Cloud Logs for which specific agent are required are deployed. +* Make sure that the Cloud Monitoring instance is deployed. + +![monitoring-agent-deployable-architecture](../../reference-architecture/deployable-architecture-logs-agent.svg) + +**NB:** This solution is not intended to be called by one or more other modules since it contains a provider configurations, meaning it is not compatible with the `for_each`, `count`, and `depends_on` arguments. For more information see [Providers Within Modules](https://developer.hashicorp.com/terraform/language/modules/develop/providers) diff --git a/solutions/fully-configurable/provider.tf b/solutions/fully-configurable/provider.tf index 63d2619c..40a74691 100644 --- a/solutions/fully-configurable/provider.tf +++ b/solutions/fully-configurable/provider.tf @@ -3,14 +3,16 @@ provider "ibm" { } provider "kubernetes" { - host = data.ibm_container_cluster_config.cluster_config.host - token = data.ibm_container_cluster_config.cluster_config.token + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate } provider "helm" { kubernetes { - host = data.ibm_container_cluster_config.cluster_config.host - token = data.ibm_container_cluster_config.cluster_config.token + host = data.ibm_container_cluster_config.cluster_config.host + token = data.ibm_container_cluster_config.cluster_config.token + cluster_ca_certificate = data.ibm_container_cluster_config.cluster_config.ca_certificate } # IBM Cloud credentials are required to authenticate to the helm repo registry { From 294ab9492428876388eebac9f4081e04408f6eb3 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 7 Apr 2025 20:42:43 +0100 Subject: [PATCH 04/21] fix: update code --- common-dev-assets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-dev-assets b/common-dev-assets index 6e39f79e..2a2281ec 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 6e39f79e1389a1ec3bbe57215573e7d4e6dd98f1 +Subproject commit 2a2281eca386901262a1d0c7b617dc07476d5944 From 9e9bbec6a3b8a0a61896b52e7d8d656c72ba9f76 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 9 Apr 2025 13:49:23 +0100 Subject: [PATCH 05/21] fix: address feedback --- .catalog-onboard-pipeline.yaml | 2 +- .github/settings.yml | 2 +- README.md | 6 +++--- ibm_catalog.json | 8 ++++---- main.tf | 9 ++++----- solutions/fully-configurable/README.md | 2 +- solutions/fully-configurable/main.tf | 6 +++--- solutions/fully-configurable/variables.tf | 18 +++++++++--------- variables.tf | 19 +++++++++---------- 9 files changed, 35 insertions(+), 37 deletions(-) diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml index a9633bcb..bb3e76d9 100644 --- a/.catalog-onboard-pipeline.yaml +++ b/.catalog-onboard-pipeline.yaml @@ -4,7 +4,7 @@ offerings: - name: deploy-arch-ibm-logs-agent kind: solution catalog_id: 7df1e4ca-d54c-4fd0-82ce-3d13247308cd - offering_id: _ + offering_id: b3fb668c-6f9d-4db3-9073-9a677ccf4394 variations: - name: fully-configurable mark_ready: true diff --git a/.github/settings.yml b/.github/settings.yml index 927feecc..bbcfdfe9 100644 --- a/.github/settings.yml +++ b/.github/settings.yml @@ -22,7 +22,7 @@ repository: # Uncomment this description property # and update the description to the current repo description. - description: "Terraform module to deploy the IBM Cloud Logs Agent (LogDNA) on a Cluster" + description: "Terraform module to deploy the IBM Cloud Logs Agent on a Cluster" # Use a comma-separated list of topics to set on the repo (ensure not to use any caps in the topic string). topics: terraform, ibm-cloud, terraform-module, core-team, logs-agent, observability, logs, agent, cloud-logs, logging-agent diff --git a/README.md b/README.md index d93efb62..03d19abd 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,9 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [chart](#input\_chart) | The name of the Helm chart to deploy. | `string` | `"logs-agent-helm"` | no | +| [chart\_repository](#input\_chart\_repository) | The repository URL for the Helm chart. | `string` | `"oci://icr.io/ibm/observe"` | no | +| [chart\_version](#input\_chart\_version) | The version of the Sysdig Helm chart to deploy. | `string` | `"1.5.1"` | no | | [cloud\_logs\_ingress\_endpoint](#input\_cloud\_logs\_ingress\_endpoint) | The host for IBM Cloud Logs ingestion. Ensure you use the ingress endpoint. See https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-endpoints_ingress. | `string` | n/a | yes | | [cloud\_logs\_ingress\_port](#input\_cloud\_logs\_ingress\_port) | The target port for the IBM Cloud Logs ingestion endpoint. The port must be 443 if you connect by using a VPE gateway, or port 3443 when you connect by using CSEs. | `number` | `3443` | no | | [cluster\_config\_endpoint\_type](#input\_cluster\_config\_endpoint\_type) | The type of endpoint to use for the cluster config access: `default`, `private`, `vpe`, or `link`. The `default` value uses the default endpoint of the cluster. | `string` | `"default"` | no | @@ -93,8 +96,6 @@ No modules. | [is\_vpc\_cluster](#input\_is\_vpc\_cluster) | Specify true if the target cluster for the agent is a VPC cluster, false if it is a classic cluster. | `bool` | `true` | no | | [logs\_agent\_additional\_log\_source\_paths](#input\_logs\_agent\_additional\_log\_source\_paths) | The list of additional log sources. By default, the Logs agent collects logs from a single source at `/var/log/containers/*.log`. | `list(string)` | `[]` | no | | [logs\_agent\_additional\_metadata](#input\_logs\_agent\_additional\_metadata) | The list of additional metadata fields to add to the routed logs. | 
list(object({
    key   = optional(string)
    value = optional(string)
  }))
| `[]` | no | -| [logs\_agent\_chart\_location](#input\_logs\_agent\_chart\_location) | The location of the Helm chart for the Logs agent. | `string` | `"oci://icr.io/ibm/observe/logs-agent-helm"` | no | -| [logs\_agent\_chart\_version](#input\_logs\_agent\_chart\_version) | The version of the helm chart to deploy. | `string` | `"1.4.2"` | no | | [logs\_agent\_enable\_scc](#input\_logs\_agent\_enable\_scc) | Whether to enable creation of Security Context Constraints in Openshift. When installing on an OpenShift cluster, this setting is mandatory to configure permissions for pods within your cluster. | `bool` | `true` | no | | [logs\_agent\_exclude\_log\_source\_paths](#input\_logs\_agent\_exclude\_log\_source\_paths) | The list of log sources to exclude. Specify the paths that the Logs agent ignores. | `list(string)` | `[]` | no | | [logs\_agent\_iam\_api\_key](#input\_logs\_agent\_iam\_api\_key) | The IBM Cloud API key for the Logs agent to authenticate and communicate with the IBM Cloud Logs. It is required if `logs_agent_iam_mode` is set to `IAMAPIKey`. | `string` | `null` | no | @@ -107,7 +108,6 @@ No modules. | [logs\_agent\_selected\_log\_source\_paths](#input\_logs\_agent\_selected\_log\_source\_paths) | The list of specific log sources paths. Logs will only be collected from the specified log source paths. If no paths are specified, it will send logs from `/var/log/containers`. | `list(string)` | `[]` | no | | [logs\_agent\_tolerations](#input\_logs\_agent\_tolerations) | List of tolerations to apply to Logs agent. The default value means a pod will run on every node. | 
list(object({
    key               = optional(string)
    operator          = optional(string)
    value             = optional(string)
    effect            = optional(string)
    tolerationSeconds = optional(number)
  }))
| 
[
  {
    "operator": "Exists"
  }
]
| no | | [logs\_agent\_trusted\_profile](#input\_logs\_agent\_trusted\_profile) | The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role. | `string` | `null` | no | -| [logs\_agent\_version](#input\_logs\_agent\_version) | The version of the Logs agent to deploy. | `string` | `"1.4.2"` | no | | [wait\_till](#input\_wait\_till) | To avoid long wait times when you run your Terraform code, you can specify the stage when you want Terraform to mark the cluster resource creation as completed. Depending on what stage you choose, the cluster creation might not be fully completed and continues to run in the background. However, your Terraform code can continue to run without waiting for the cluster to be fully created. Supported args are `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady` and `Normal` | `string` | `"Normal"` | no | | [wait\_till\_timeout](#input\_wait\_till\_timeout) | Timeout for wait\_till in minutes. | `number` | `90` | no | diff --git a/ibm_catalog.json b/ibm_catalog.json index 07622214..353730df 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -1,7 +1,7 @@ { "products": [ { - "label": "Cloud automation for Logs Agent", + "label": "Cloud automation for Logs agent", "name": "deploy-arch-ibm-logs-agent", "product_kind": "solution", "tags": [ @@ -107,13 +107,13 @@ "required": true }, { - "key": "logs_agent_chart_location" + "key": "chart" }, { - "key": "logs_agent_chart_version" + "key": "chart_repository" }, { - "key": "logs_agent_version" + "key": "chart_version" }, { "key": "logs_agent_resources" diff --git a/main.tf b/main.tf index 92ee5a0c..99284341 100644 --- a/main.tf +++ b/main.tf @@ -24,8 +24,6 @@ data "ibm_container_cluster_config" "cluster_config" { } locals { - logs_agent_chart_location = var.logs_agent_chart_location - logs_agent_version = var.logs_agent_version != null ? var.logs_agent_version : var.logs_agent_chart_version # use logs_agent_chart_version somewhere so pre-commit passes until its needed logs_agent_selected_log_source_paths = distinct(concat([for namespace in var.logs_agent_log_source_namespaces : "/var/log/containers/*_${namespace}_*.log"], var.logs_agent_selected_log_source_paths)) logs_agent_iam_api_key = var.logs_agent_iam_api_key != null ? var.logs_agent_iam_api_key : "" logs_agent_trusted_profile = var.logs_agent_trusted_profile != null ? var.logs_agent_trusted_profile : "" @@ -39,8 +37,9 @@ locals { resource "helm_release" "logs_agent" { name = var.logs_agent_name - chart = local.logs_agent_chart_location - version = local.logs_agent_version + chart = var.chart + repository = var.chart_repository + version = var.chart_version namespace = var.logs_agent_namespace create_namespace = true timeout = 1200 @@ -56,7 +55,7 @@ resource "helm_release" "logs_agent" { set { name = "image.version" type = "string" - value = local.logs_agent_version + value = var.chart_version } set { name = "env.ingestionHost" diff --git a/solutions/fully-configurable/README.md b/solutions/fully-configurable/README.md index 6f19f3c6..12be964a 100644 --- a/solutions/fully-configurable/README.md +++ b/solutions/fully-configurable/README.md @@ -1,4 +1,4 @@ -# Cloud automation for Logs Agent +# Cloud automation for Logs agent This architecture deploys the following monitoring agent on a Red Hat OpenShift cluster: diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf index 26518f13..b0ec29ef 100644 --- a/solutions/fully-configurable/main.tf +++ b/solutions/fully-configurable/main.tf @@ -20,9 +20,9 @@ module "logs_agent" { cluster_resource_group_id = var.cluster_resource_group_id cluster_config_endpoint_type = local.cluster_config_endpoint_type # Logs Agent - logs_agent_chart_location = var.logs_agent_chart_location - logs_agent_chart_version = var.logs_agent_chart_version - logs_agent_version = var.logs_agent_version + chart = var.chart + chart_repository = var.chart_repository + chart_version = var.chart_version logs_agent_name = var.logs_agent_name logs_agent_namespace = var.logs_agent_namespace logs_agent_trusted_profile = var.logs_agent_trusted_profile_id diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index 41eee762..b893577f 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -47,24 +47,24 @@ variable "wait_till_timeout" { # Logs Agent variables ############################################################################## -variable "logs_agent_version" { - description = "The version of the Logs agent to deploy." +variable "chart" { + description = "The name of the Helm chart to deploy." type = string - default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + default = "logs-agent-helm" # Replace with the actual chart name if different nullable = false } -variable "logs_agent_chart_version" { - description = "The version of the helm chart to deploy." +variable "chart_repository" { + description = "The repository URL for the Helm chart." type = string - default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + default = "oci://icr.io/ibm/observe" # Replace with the actual repository URL if different nullable = false } -variable "logs_agent_chart_location" { - description = "The location of the Helm chart for the Logs agent." +variable "chart_version" { + description = "The version of the Sysdig Helm chart to deploy." type = string - default = "oci://icr.io/ibm/observe/logs-agent-helm" + default = "1.5.1" # datasource: icr.io/ibm/observe/logs-agent-helm } variable "logs_agent_name" { diff --git a/variables.tf b/variables.tf index 5d2a8e6f..34d8d73c 100644 --- a/variables.tf +++ b/variables.tf @@ -55,25 +55,24 @@ variable "wait_till_timeout" { # Logs Agent variables ############################################################################## -variable "logs_agent_version" { - description = "The version of the Logs agent to deploy." +variable "chart" { + description = "The name of the Helm chart to deploy." type = string - default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + default = "logs-agent-helm" # Replace with the actual chart name if different nullable = false } -variable "logs_agent_chart_version" { - description = "The version of the helm chart to deploy." +variable "chart_repository" { + description = "The repository URL for the Helm chart." type = string - default = "1.4.2" # datasource: icr.io/ibm/observe/logs-agent-helm + default = "oci://icr.io/ibm/observe" # Replace with the actual repository URL if different nullable = false } -variable "logs_agent_chart_location" { - description = "The location of the Helm chart for the Logs agent." +variable "chart_version" { + description = "The version of the Sysdig Helm chart to deploy." type = string - default = "oci://icr.io/ibm/observe/logs-agent-helm" - nullable = false + default = "1.5.1" # datasource: icr.io/ibm/observe/logs-agent-helm } variable "logs_agent_name" { From 99f611815a70567162ada95e3de4068daed04335 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 9 Apr 2025 14:31:10 +0100 Subject: [PATCH 06/21] fix: address feedback --- .../scripts/post-validation-deploy-slz-roks-and-obs-instances.sh | 0 tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh mode change 100644 => 100755 tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh diff --git a/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh old mode 100644 new mode 100755 diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh old mode 100644 new mode 100755 From cfe48a45329cb7030cd1d5ca8b6e72f2b92a83ad Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 14 Apr 2025 22:52:52 +0100 Subject: [PATCH 07/21] fix: address feedback --- README.md | 7 ++++--- ibm_catalog.json | 9 ++++++--- main.tf | 8 ++++---- solutions/fully-configurable/main.tf | 7 ++++--- solutions/fully-configurable/variables.tf | 18 +++++++++++++----- variables.tf | 19 +++++++++++++------ 6 files changed, 44 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 03d19abd..acd56eac 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,6 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [chart](#input\_chart) | The name of the Helm chart to deploy. | `string` | `"logs-agent-helm"` | no | -| [chart\_repository](#input\_chart\_repository) | The repository URL for the Helm chart. | `string` | `"oci://icr.io/ibm/observe"` | no | -| [chart\_version](#input\_chart\_version) | The version of the Sysdig Helm chart to deploy. | `string` | `"1.5.1"` | no | | [cloud\_logs\_ingress\_endpoint](#input\_cloud\_logs\_ingress\_endpoint) | The host for IBM Cloud Logs ingestion. Ensure you use the ingress endpoint. See https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-endpoints_ingress. | `string` | n/a | yes | | [cloud\_logs\_ingress\_port](#input\_cloud\_logs\_ingress\_port) | The target port for the IBM Cloud Logs ingestion endpoint. The port must be 443 if you connect by using a VPE gateway, or port 3443 when you connect by using CSEs. | `number` | `3443` | no | | [cluster\_config\_endpoint\_type](#input\_cluster\_config\_endpoint\_type) | The type of endpoint to use for the cluster config access: `default`, `private`, `vpe`, or `link`. The `default` value uses the default endpoint of the cluster. | `string` | `"default"` | no | @@ -96,11 +93,15 @@ No modules. | [is\_vpc\_cluster](#input\_is\_vpc\_cluster) | Specify true if the target cluster for the agent is a VPC cluster, false if it is a classic cluster. | `bool` | `true` | no | | [logs\_agent\_additional\_log\_source\_paths](#input\_logs\_agent\_additional\_log\_source\_paths) | The list of additional log sources. By default, the Logs agent collects logs from a single source at `/var/log/containers/*.log`. | `list(string)` | `[]` | no | | [logs\_agent\_additional\_metadata](#input\_logs\_agent\_additional\_metadata) | The list of additional metadata fields to add to the routed logs. | 
list(object({
    key   = optional(string)
    value = optional(string)
  }))
| `[]` | no | +| [logs\_agent\_chart](#input\_logs\_agent\_chart) | The name of the Helm chart to deploy. | `string` | `"logs-agent-helm"` | no | +| [logs\_agent\_chart\_location](#input\_logs\_agent\_chart\_location) | The location of the Logs agent helm chart. | `string` | `"oci://icr.io/ibm/observe"` | no | +| [logs\_agent\_chart\_version](#input\_logs\_agent\_chart\_version) | The version of the Helm chart to deploy. | `string` | `"1.5.1"` | no | | [logs\_agent\_enable\_scc](#input\_logs\_agent\_enable\_scc) | Whether to enable creation of Security Context Constraints in Openshift. When installing on an OpenShift cluster, this setting is mandatory to configure permissions for pods within your cluster. | `bool` | `true` | no | | [logs\_agent\_exclude\_log\_source\_paths](#input\_logs\_agent\_exclude\_log\_source\_paths) | The list of log sources to exclude. Specify the paths that the Logs agent ignores. | `list(string)` | `[]` | no | | [logs\_agent\_iam\_api\_key](#input\_logs\_agent\_iam\_api\_key) | The IBM Cloud API key for the Logs agent to authenticate and communicate with the IBM Cloud Logs. It is required if `logs_agent_iam_mode` is set to `IAMAPIKey`. | `string` | `null` | no | | [logs\_agent\_iam\_environment](#input\_logs\_agent\_iam\_environment) | IAM authentication Environment: `Production` or `PrivateProduction` or `Staging` or `PrivateStaging`. `Production` specifies the public endpoint & `PrivateProduction` specifies the private endpoint. | `string` | `"PrivateProduction"` | no | | [logs\_agent\_iam\_mode](#input\_logs\_agent\_iam\_mode) | IAM authentication mode: `TrustedProfile` or `IAMAPIKey`. | `string` | `"TrustedProfile"` | no | +| [logs\_agent\_image\_version](#input\_logs\_agent\_image\_version) | The version of the Logs agent image to deploy. | `string` | `"1.5.1"` | no | | [logs\_agent\_log\_source\_namespaces](#input\_logs\_agent\_log\_source\_namespaces) | The list of namespaces from which logs should be forwarded by agent. If namespaces are not listed, logs from all namespaces will be sent. | `list(string)` | `[]` | no | | [logs\_agent\_name](#input\_logs\_agent\_name) | The name of the Logs agent. The name is used in all Kubernetes and Helm resources in the cluster. | `string` | `"logs-agent"` | no | | [logs\_agent\_namespace](#input\_logs\_agent\_namespace) | The namespace where the Logs agent is deployed. The default value is `ibm-agent`. | `string` | `"ibm-agent"` | no | diff --git a/ibm_catalog.json b/ibm_catalog.json index 353730df..aa99d844 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -107,13 +107,16 @@ "required": true }, { - "key": "chart" + "key": "logs_agent_chart" }, { - "key": "chart_repository" + "key": "logs_agent_chart_location" }, { - "key": "chart_version" + "key": "logs_agent_chart_version" + }, + { + "key": "logs_agent_image_version" }, { "key": "logs_agent_resources" diff --git a/main.tf b/main.tf index 99284341..7bf5659a 100644 --- a/main.tf +++ b/main.tf @@ -37,9 +37,9 @@ locals { resource "helm_release" "logs_agent" { name = var.logs_agent_name - chart = var.chart - repository = var.chart_repository - version = var.chart_version + chart = var.logs_agent_chart + repository = var.logs_agent_chart_location + version = var.logs_agent_chart_version namespace = var.logs_agent_namespace create_namespace = true timeout = 1200 @@ -55,7 +55,7 @@ resource "helm_release" "logs_agent" { set { name = "image.version" type = "string" - value = var.chart_version + value = var.logs_agent_image_version } set { name = "env.ingestionHost" diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf index b0ec29ef..97323a98 100644 --- a/solutions/fully-configurable/main.tf +++ b/solutions/fully-configurable/main.tf @@ -20,9 +20,10 @@ module "logs_agent" { cluster_resource_group_id = var.cluster_resource_group_id cluster_config_endpoint_type = local.cluster_config_endpoint_type # Logs Agent - chart = var.chart - chart_repository = var.chart_repository - chart_version = var.chart_version + logs_agent_chart = var.logs_agent_chart + logs_agent_chart_location = var.logs_agent_chart_location + logs_agent_chart_version = var.logs_agent_chart_version + logs_agent_image_version = var.logs_agent_image_version logs_agent_name = var.logs_agent_name logs_agent_namespace = var.logs_agent_namespace logs_agent_trusted_profile = var.logs_agent_trusted_profile_id diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index b893577f..eac50d80 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -47,24 +47,32 @@ variable "wait_till_timeout" { # Logs Agent variables ############################################################################## -variable "chart" { +variable "logs_agent_chart" { description = "The name of the Helm chart to deploy." type = string default = "logs-agent-helm" # Replace with the actual chart name if different nullable = false } -variable "chart_repository" { - description = "The repository URL for the Helm chart." +variable "logs_agent_chart_location" { + description = "The location of the Logs agent helm chart." type = string default = "oci://icr.io/ibm/observe" # Replace with the actual repository URL if different nullable = false } -variable "chart_version" { - description = "The version of the Sysdig Helm chart to deploy." +variable "logs_agent_chart_version" { + description = "The version of the Helm chart to deploy." type = string default = "1.5.1" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false +} + +variable "logs_agent_image_version" { + description = "The version of the Logs agent image to deploy." + type = string + default = "1.5.1" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false } variable "logs_agent_name" { diff --git a/variables.tf b/variables.tf index 34d8d73c..140273b0 100644 --- a/variables.tf +++ b/variables.tf @@ -55,24 +55,32 @@ variable "wait_till_timeout" { # Logs Agent variables ############################################################################## -variable "chart" { +variable "logs_agent_chart" { description = "The name of the Helm chart to deploy." type = string default = "logs-agent-helm" # Replace with the actual chart name if different nullable = false } -variable "chart_repository" { - description = "The repository URL for the Helm chart." +variable "logs_agent_chart_location" { + description = "The location of the Logs agent helm chart." type = string default = "oci://icr.io/ibm/observe" # Replace with the actual repository URL if different nullable = false } -variable "chart_version" { - description = "The version of the Sysdig Helm chart to deploy." +variable "logs_agent_chart_version" { + description = "The version of the Helm chart to deploy." type = string default = "1.5.1" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false +} + +variable "logs_agent_image_version" { + description = "The version of the Logs agent image to deploy." + type = string + default = "1.5.1" # datasource: icr.io/ibm/observe/logs-agent-helm + nullable = false } variable "logs_agent_name" { @@ -99,7 +107,6 @@ variable "logs_agent_trusted_profile" { } } - variable "logs_agent_iam_api_key" { type = string description = "The IBM Cloud API key for the Logs agent to authenticate and communicate with the IBM Cloud Logs. It is required if `logs_agent_iam_mode` is set to `IAMAPIKey`." From 9cdb0eb83b0c0e16c153ce8e0abab33cd30e5907 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 28 Apr 2025 21:01:27 +0100 Subject: [PATCH 08/21] fix: making in-line with monitoring repo --- README.md | 50 +++++++++++++++---- ibm_catalog.json | 6 +-- ...ation-deploy-slz-roks-and-obs-instances.sh | 5 +- ...ation-deploy-slz-roks-and-obs-instances.sh | 8 +-- 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index acd56eac..e78b7f8c 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,29 @@ -# Logs Agent module - - -[![Incubating (Not yet consumable)](https://img.shields.io/badge/status-Incubating%20(Not%20yet%20consumable)-red)](https://terraform-ibm-modules.github.io/documentation/#/badge-status) -[![latest release](https://img.shields.io/github/v/release/terraform-ibm-modules/terraform-ibm-logs-agent?logo=GitHub&sort=semver)](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/releases/latest) + +# Terraform IBM Logs agent module + +[![Graduated (Supported)](https://img.shields.io/badge/Status-Graduated%20(Supported)-brightgreen)](https://terraform-ibm-modules.github.io/documentation/#/badge-status) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://github.com/pre-commit/pre-commit) +[![latest release](https://img.shields.io/github/v/release/terraform-ibm-modules/terraform-ibm-logs-agent?logo=GitHub&sort=semver)](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/releases/latest) [![Renovate enabled](https://img.shields.io/badge/renovate-enabled-brightgreen.svg)](https://renovatebot.com/) [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) -You can use this module to deploy Logs Agent in your cluster. The agent can collect and route application logs from a Red Hat OpenShift or Kubernetes cluster to an IBM Cloud Logs of your choice. +This module deploys the following logs agent to an IBM Cloud Red Hat OpenShift Container Platform or Kubernetes cluster: + +- [Logs agent](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-about) + + + +## Overview +* [terraform-ibm-logs-agent](#terraform-ibm-logs-agent) +* [Examples](./examples) + * [Logs agent on Kubernetes using CSE ingress endpoint with an apikey](./examples/logs-agent-iks) + * [Logs agent on OCP using VPE ingress endpoint with a Trusted Profile](./examples/logs-agent-ocp) +* [Contributing](#contributing) + + +## terraform-ibm-logs-agent -## Usage +### Usage ```hcl # ############################################################################ @@ -59,6 +69,16 @@ module "logs_agent_module" { } ``` +### Required IAM access policies +You need the following permissions to run this module. + +- Service + - **Resource group only** + - `Viewer` access on the specific resource group + - **Kubernetes** service + - `Viewer` platform access + - `Manager` service access + ### Requirements @@ -116,3 +136,11 @@ No modules. No outputs. + + + +## Contributing + +You can report issues and request features for this module in GitHub issues in the module repo. See [Report an issue or request a feature](https://github.com/terraform-ibm-modules/.github/blob/main/.github/SUPPORT.md). + +To set up your local development environment, see [Local development setup](https://terraform-ibm-modules.github.io/documentation/#/local-dev-setup) in the project documentation. diff --git a/ibm_catalog.json b/ibm_catalog.json index aa99d844..643e1d11 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -22,9 +22,9 @@ ], "provider_name": "IBM", "short_description": "Deploys IBM Logs Agent to a cluster", - "long_description": "Solutions that support deploying IBM Logs Agent for logging.", + "long_description": "Solution that support deploying IBM Logs Agent for logging.", "offering_docs_url": "https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/README.md", - "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/main/images/logs-icon.svg", + "offering_icon_url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/refs/heads/main/images/logs-icon.svg", "flavors": [ { "label": "Fully Configurable", @@ -54,7 +54,7 @@ "diagrams": [ { "diagram": { - "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/reference-architecture/deployable-architecture-logs-agent.svg", + "url": "https://raw.githubusercontent.com/terraform-ibm-modules/terraform-ibm-logs-agent/refs/heads/main/reference-architecture/deployable-architecture-logs-agent.svg", "caption": "Logs Agent on a cluster", "type": "image/svg+xml" }, diff --git a/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh index ba0f6092..b9f8fc73 100755 --- a/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh +++ b/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh @@ -1,8 +1,7 @@ #! /bin/bash ######################################################################################################################## -## This script is used by the catalog pipeline to destroy the SLZ OCP Cluster, which was provisioned as a ## -## prerequisite for the WAS extension that is published to the catalog ## +## This script is used by the catalog pipeline to destroy prerequisite resource required for catalog validation ## ######################################################################################################################## set -e @@ -12,7 +11,7 @@ TF_VARS_FILE="terraform.tfvars" ( cd ${TERRAFORM_SOURCE_DIR} - echo "Destroying prerequisite SLZ OCP Cluster and Observability instances .." + echo "Destroying prerequisite SLZ OCP Cluster and Logs instance.." terraform destroy -input=false -auto-approve -var-file=${TF_VARS_FILE} || exit 1 rm -f "${TF_VARS_FILE}" diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh index b212d7bb..0c59a5b5 100755 --- a/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh +++ b/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh @@ -1,8 +1,8 @@ #! /bin/bash ############################################################################################################ -## This script is used by the catalog pipeline to deploy the SLZ ROKS and Observability instances, -## which are the prerequisites for the Logs Agent extension. +## This script is used by the catalog pipeline to deploy the OCP and Monitoring instances, +## which are the prerequisites for the Monitoring Agent DA. ############################################################################################################ set -e @@ -16,7 +16,7 @@ TF_VARS_FILE="terraform.tfvars" ( cwd=$(pwd) cd ${TERRAFORM_SOURCE_DIR} - echo "Provisioning prerequisite SLZ ROKS CLUSTER and Observability Instances .." + echo "Provisioning prerequisite SLZ ROKS CLUSTER and Logs Instance.." terraform init || exit 1 # $VALIDATION_APIKEY is available in the catalog runtime { @@ -36,7 +36,7 @@ TF_VARS_FILE="terraform.tfvars" cloud_logs_ingress_endpoint_var_name="cloud_logs_ingress_endpoint" cloud_logs_ingress_endpoint_value=$(terraform output -state=terraform.tfstate -raw cloud_logs_ingress_private_endpoint) - echo "Appending '${cluster_id_var_name}' and '${region_var_name}' input variable values to ${JSON_FILE}.." + echo "Appending '${region_var_name}', '${cluster_id_var_name}' '${cluster_resource_group_id_var_name}', '${logs_agent_trusted_profile_var_name}', and '${cloud_logs_ingress_endpoint_var_name}' input variable values to ${JSON_FILE}.." cd "${cwd}" jq -r --arg region_var_name "${region_var_name}" \ From 0068224c0b968625667c061d4ebc9f2013f0af2a Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 28 Apr 2025 21:02:57 +0100 Subject: [PATCH 09/21] fix: making in-line with monitoring repo --- .catalog-onboard-pipeline.yaml | 4 ++-- ... => post-validation-deploy-slz-roks-and-logs-instances.sh} | 0 ...h => pre-validation-deploy-slz-roks-and-logs-instances.sh} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/scripts/{post-validation-deploy-slz-roks-and-obs-instances.sh => post-validation-deploy-slz-roks-and-logs-instances.sh} (100%) rename tests/scripts/{pre-validation-deploy-slz-roks-and-obs-instances.sh => pre-validation-deploy-slz-roks-and-logs-instances.sh} (100%) diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml index bb3e76d9..60f32292 100644 --- a/.catalog-onboard-pipeline.yaml +++ b/.catalog-onboard-pipeline.yaml @@ -9,5 +9,5 @@ offerings: - name: fully-configurable mark_ready: true install_type: fullstack - pre_validation: "tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh" - post_validation: "tests/scripts/post-validation-destroy-slz-roks-and-obs-instances.sh" + pre_validation: "tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh" + post_validation: "tests/scripts/post-validation-destroy-slz-roks-and-logs-instances.sh" diff --git a/tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh similarity index 100% rename from tests/scripts/post-validation-deploy-slz-roks-and-obs-instances.sh rename to tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh b/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh similarity index 100% rename from tests/scripts/pre-validation-deploy-slz-roks-and-obs-instances.sh rename to tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh From c5dd363c30f027e93ee8c2b487075ee509447480 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 28 Apr 2025 21:18:30 +0100 Subject: [PATCH 10/21] fix: required --- ibm_catalog.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index 643e1d11..bcd49d11 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -73,7 +73,8 @@ "type": "cluster_var", "grouping": "deployment", "original_grouping": "deployment" - } + }, + "required": true }, { "key": "cluster_resource_group_id", @@ -84,7 +85,8 @@ "config_constraints": { "identifier": "rg_id" } - } + }, + "required": true }, { "key": "is_vpc_cluster", From 1e5ffc84cb97b5a347d226d236d28abe74e272ae Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 28 Apr 2025 21:52:11 +0100 Subject: [PATCH 11/21] fix: required --- solutions/fully-configurable/variables.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index eac50d80..2a731ae0 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -216,4 +216,8 @@ variable "cloud_logs_ingress_port" { type = number default = 3443 description = "The target port for the IBM Cloud Logs ingestion endpoint. The port must be 443 if you connect by using a VPE gateway, or port 3443 when you connect by using CSEs." + validation { + error_message = "The Logs Routing supertenant ingestion port can only be `3443` or `443`." + condition = contains([3443, 443], var.cloud_logs_ingress_port) + } } From b36dd5064e3ddb5affa9cd9db00189b9baa2eba8 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 29 Apr 2025 11:09:03 +0100 Subject: [PATCH 12/21] fix: update ibm catalog --- ibm_catalog.json | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index bcd49d11..6edf1b56 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -78,14 +78,6 @@ }, { "key": "cluster_resource_group_id", - "custom_config": { - "type": "resource_group", - "grouping": "deployment", - "original_grouping": "deployment", - "config_constraints": { - "identifier": "rg_id" - } - }, "required": true }, { From 69af5fdfac02f4a5fe492d94d153097f2700158c Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 29 Apr 2025 11:26:01 +0100 Subject: [PATCH 13/21] fix: update permissions --- ibm_catalog.json | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ibm_catalog.json b/ibm_catalog.json index 6edf1b56..e90b5914 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -33,10 +33,21 @@ "compliance": {}, "iam_permissions": [ { - "service_name": "containers-kubernetes", + "service_name": "sysdig-monitor", "role_crns": [ - "crn:v1:bluemix:public:iam::::serviceRole:Manager", - "crn:v1:bluemix:public:iam::::role:Viewer" + "crn:v1:bluemix:public:iam::::serviceRole:Manager" + ] + }, + { + "service_name": "logs", + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" + ] + }, + { + "service_name": "metrics-router", + "role_crns": [ + "crn:v1:bluemix:public:iam::::role:Editor" ] } ], From 59b615f6800b462c01c3cf71ec7eb4814bef8b17 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 30 Apr 2025 10:03:31 +0100 Subject: [PATCH 14/21] fix: update code --- common-dev-assets | 2 +- ibm_catalog.json | 17 +--- solutions/fully-configurable/DA-types.md | 2 +- solutions/fully-configurable/variables.tf | 6 +- tests/resources/main.tf | 93 +++++++++++++++---- tests/resources/outputs.tf | 11 +-- ...tion-deploy-slz-roks-and-logs-instances.sh | 11 +-- 7 files changed, 88 insertions(+), 54 deletions(-) diff --git a/common-dev-assets b/common-dev-assets index 252e9276..1f72de03 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 252e927652827bf9ecb14b950f5bcf3ccd1f69aa +Subproject commit 1f72de03a36f8ee5522aaacf0ba410b6481122e9 diff --git a/ibm_catalog.json b/ibm_catalog.json index e90b5914..6edf1b56 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -33,21 +33,10 @@ "compliance": {}, "iam_permissions": [ { - "service_name": "sysdig-monitor", + "service_name": "containers-kubernetes", "role_crns": [ - "crn:v1:bluemix:public:iam::::serviceRole:Manager" - ] - }, - { - "service_name": "logs", - "role_crns": [ - "crn:v1:bluemix:public:iam::::role:Editor" - ] - }, - { - "service_name": "metrics-router", - "role_crns": [ - "crn:v1:bluemix:public:iam::::role:Editor" + "crn:v1:bluemix:public:iam::::serviceRole:Manager", + "crn:v1:bluemix:public:iam::::role:Viewer" ] } ], diff --git a/solutions/fully-configurable/DA-types.md b/solutions/fully-configurable/DA-types.md index 283ffb79..c6c4c10b 100644 --- a/solutions/fully-configurable/DA-types.md +++ b/solutions/fully-configurable/DA-types.md @@ -74,7 +74,7 @@ The `logs_agent_additional_metadata` variable is used to configure additional me # Configuring Logs Agent Resources -When you deploy the IBM Logs agent using the `terraform-ibm-logs-agent` module, you can configure the resource requests and limits for the logs agent pods by using the `logs_agent_resources` variable. This variable allows you to specify the CPU and memory resources allocated to the logs agent. +When you deploy the IBM Logs agent using the `terraform-ibm-logs-agent` module, you can configure the resource requests and limits for the logs agent pods by using the `logs_agent_resources` variable. This variable allows you to specify the CPU and memory resources allocated to the logs agent. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources). ### Options for `logs_agent_resources` - `requests` (optional): Specifies the minimum amount of resources required. Includes: diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index 2a731ae0..0fe7860d 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -107,7 +107,7 @@ variable "logs_agent_iam_api_key" { } variable "logs_agent_tolerations" { - description = "List of tolerations to apply to Logs agent. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md)." + description = "List of tolerations to apply to Logs agent. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md#configuring-logs-agent-tolerations)." type = list(object({ key = optional(string) operator = optional(string) @@ -121,7 +121,7 @@ variable "logs_agent_tolerations" { } variable "logs_agent_resources" { - description = "The resources configuration for cpu/memory/storage. Learn more [here](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources) and [here](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md)." + description = "The resources configuration for cpu/memory/storage. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/DA-types.md#configuring-logs-agent-resources)." type = object({ limits = object({ cpu = string @@ -193,7 +193,7 @@ variable "logs_agent_iam_environment" { } variable "logs_agent_additional_metadata" { - description = "The list of additional metadata fields to add to the routed logs. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/tree/main/solutions/fully-configurable/DA-types.md)." + description = "The list of additional metadata fields to add to the routed logs. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-logs-agent/blob/main/solutions/fully-configurable/DA-types.md#configuring-logs-agent-additional-metadata)." type = list(object({ key = optional(string) value = optional(string) diff --git a/tests/resources/main.tf b/tests/resources/main.tf index 6297d354..7b409117 100644 --- a/tests/resources/main.tf +++ b/tests/resources/main.tf @@ -10,21 +10,81 @@ module "resource_group" { existing_resource_group_name = var.resource_group } -############################################################################## -# SLZ ROKS Pattern -############################################################################## +######################################################################################################################## +# VPC + Subnet + Public Gateway +# +# NOTE: This is a very simple VPC with single subnet in a single zone with a public gateway enabled, that will allow +# all traffic ingress/egress by default. +# For production use cases this would need to be enhanced by adding more subnets and zones for resiliency, and +# ACLs/Security Groups for network security. +######################################################################################################################## + +resource "ibm_is_vpc" "vpc" { + name = "${var.prefix}-vpc" + resource_group = module.resource_group.resource_group_id + address_prefix_management = "auto" + tags = var.resource_tags +} + +resource "ibm_is_public_gateway" "gateway" { + name = "${var.prefix}-gateway-1" + vpc = ibm_is_vpc.vpc.id + resource_group = module.resource_group.resource_group_id + zone = "${var.region}-1" +} + +resource "ibm_is_subnet" "subnet_zone_1" { + name = "${var.prefix}-subnet-1" + vpc = ibm_is_vpc.vpc.id + resource_group = module.resource_group.resource_group_id + zone = "${var.region}-1" + total_ipv4_address_count = 256 + public_gateway = ibm_is_public_gateway.gateway.id +} + +######################################################################################################################## +# OCP VPC cluster (single zone) +######################################################################################################################## + +locals { + cluster_vpc_subnets = { + default = [ + { + id = ibm_is_subnet.subnet_zone_1.id + cidr_block = ibm_is_subnet.subnet_zone_1.ipv4_cidr_block + zone = ibm_is_subnet.subnet_zone_1.zone + } + ] + } -module "landing_zone" { - source = "git::https://github.com/terraform-ibm-modules/terraform-ibm-landing-zone//patterns//roks//module?ref=v7.3.0" + worker_pools = [ + { + subnet_prefix = "default" + pool_name = "default" # ibm_container_vpc_cluster automatically names default pool "default" (See https://github.com/IBM-Cloud/terraform-provider-ibm/issues/2849) + machine_type = "bx2.4x16" + workers_per_zone = 2 # minimum of 2 is allowed when using single zone + operating_system = "REDHAT_8_64" + } + ] +} + +locals { + cluster_name = "${var.prefix}-cluster" +} + +module "ocp_base" { + source = "terraform-ibm-modules/base-ocp-vpc/ibm" + version = "3.46.11" + resource_group_id = module.resource_group.resource_group_id region = var.region - prefix = var.prefix tags = var.resource_tags - add_atracker_route = false - enable_transit_gateway = false - cluster_force_delete_storage = true - verify_cluster_network_readiness = false - use_ibm_cloud_private_api_endpoints = false - ignore_vpcs_for_cluster_deployment = ["management"] + cluster_name = local.cluster_name + force_delete_storage = true + vpc_id = ibm_is_vpc.vpc.id + vpc_subnets = local.cluster_vpc_subnets + worker_pools = local.worker_pools + access_tags = [] + disable_outbound_traffic_protection = true # set as True to enable outbound traffic } ############################################################################## @@ -78,15 +138,10 @@ module "buckets" { # - Cloud Logs instance ############################################################################## -locals { - cluster_resource_group_id = module.landing_zone.cluster_data["${var.prefix}-workload-cluster"].resource_group_id - cluster_crn = module.landing_zone.cluster_data["${var.prefix}-workload-cluster"].crn -} - module "cloud_logs" { source = "terraform-ibm-modules/cloud-logs/ibm" version = "1.0.0" - resource_group_id = local.cluster_resource_group_id + resource_group_id = module.ocp_base.resource_group_id region = var.region instance_name = "${var.prefix}-cloud-logs" resource_tags = var.resource_tags @@ -130,7 +185,7 @@ module "trusted_profile" { trusted_profile_links = [{ cr_type = "ROKS_SA" links = [{ - crn = local.cluster_crn + crn = module.ocp_base.cluster_crn namespace = local.logs_agent_namespace name = local.logs_agent_name }] diff --git a/tests/resources/outputs.tf b/tests/resources/outputs.tf index bbc65ebb..c14e6220 100644 --- a/tests/resources/outputs.tf +++ b/tests/resources/outputs.tf @@ -12,23 +12,18 @@ output "prefix" { description = "prefix" } -output "cluster_data" { - value = module.landing_zone.cluster_data - description = "Details of OCP cluster." -} - output "workload_cluster_id" { - value = module.landing_zone.workload_cluster_id + value = module.ocp_base.cluster_id description = "ID of the workload cluster." } output "workload_cluster_crn" { - value = local.cluster_crn + value = module.ocp_base.cluster_crn description = "CRN of the workload cluster." } output "cluster_resource_group_id" { - value = local.cluster_resource_group_id + value = module.ocp_base.resource_group_id description = "Resource group ID of the workload cluster." } diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh b/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh index 0c59a5b5..c9169036 100755 --- a/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh +++ b/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh @@ -10,7 +10,6 @@ set -e DA_DIR="solutions/fully-configurable" TERRAFORM_SOURCE_DIR="tests/resources" JSON_FILE="${DA_DIR}/catalogValidationValues.json" -REGION="us-south" TF_VARS_FILE="terraform.tfvars" ( @@ -21,12 +20,10 @@ TF_VARS_FILE="terraform.tfvars" # $VALIDATION_APIKEY is available in the catalog runtime { echo "ibmcloud_api_key=\"${VALIDATION_APIKEY}\"" - echo "region=\"${REGION}\"" echo "prefix=\"slz-$(openssl rand -hex 2)\"" } >> ${TF_VARS_FILE} terraform apply -input=false -auto-approve -var-file=${TF_VARS_FILE} || exit 1 - region_var_name="region" cluster_id_var_name="cluster_id" cluster_id_value=$(terraform output -state=terraform.tfstate -raw workload_cluster_id) cluster_resource_group_id_var_name="cluster_resource_group_id" @@ -36,12 +33,10 @@ TF_VARS_FILE="terraform.tfvars" cloud_logs_ingress_endpoint_var_name="cloud_logs_ingress_endpoint" cloud_logs_ingress_endpoint_value=$(terraform output -state=terraform.tfstate -raw cloud_logs_ingress_private_endpoint) - echo "Appending '${region_var_name}', '${cluster_id_var_name}' '${cluster_resource_group_id_var_name}', '${logs_agent_trusted_profile_var_name}', and '${cloud_logs_ingress_endpoint_var_name}' input variable values to ${JSON_FILE}.." + echo "Appending '${cluster_id_var_name}' '${cluster_resource_group_id_var_name}', '${logs_agent_trusted_profile_var_name}', and '${cloud_logs_ingress_endpoint_var_name}' input variable values to ${JSON_FILE}.." cd "${cwd}" - jq -r --arg region_var_name "${region_var_name}" \ - --arg region_var_value "${REGION}" \ - --arg cluster_id_var_name "${cluster_id_var_name}" \ + jq -r --arg cluster_id_var_name "${cluster_id_var_name}" \ --arg cluster_id_value "${cluster_id_value}" \ --arg cluster_resource_group_id_var_name "${cluster_resource_group_id_var_name}" \ --arg cluster_resource_group_id_value "${cluster_resource_group_id_value}" \ @@ -49,7 +44,7 @@ TF_VARS_FILE="terraform.tfvars" --arg logs_agent_trusted_profile_value "${logs_agent_trusted_profile_value}" \ --arg cloud_logs_ingress_endpoint_var_name "${cloud_logs_ingress_endpoint_var_name}" \ --arg cloud_logs_ingress_endpoint_value "${cloud_logs_ingress_endpoint_value}" \ - '. + {($region_var_name): $region_var_value, ($cluster_id_var_name): $cluster_id_value, ($cluster_resource_group_id_var_name): $cluster_resource_group_id_value, ($logs_agent_trusted_profile_var_name): $logs_agent_trusted_profile_value, ($cloud_logs_ingress_endpoint_var_name): $cloud_logs_ingress_endpoint_value}' "${JSON_FILE}" > tmpfile && mv tmpfile "${JSON_FILE}" || exit 1 + '. + {($cluster_id_var_name): $cluster_id_value, ($cluster_resource_group_id_var_name): $cluster_resource_group_id_value, ($logs_agent_trusted_profile_var_name): $logs_agent_trusted_profile_value, ($cloud_logs_ingress_endpoint_var_name): $cloud_logs_ingress_endpoint_value}' "${JSON_FILE}" > tmpfile && mv tmpfile "${JSON_FILE}" || exit 1 echo "Pre-validation complete successfully" ) From 8b5bb946eb97cbd9c25c5afc5df1caf7954eab60 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Wed, 30 Apr 2025 10:07:47 +0100 Subject: [PATCH 15/21] fix: update code --- .catalog-onboard-pipeline.yaml | 4 ++-- common-dev-assets | 2 +- tests/pr_test.go | 12 ++++++------ tests/resources/outputs.tf | 2 +- ...-validation-deploy-slz-roks-and-logs-instances.sh | 2 +- ...-validation-deploy-slz-roks-and-logs-instances.sh | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.catalog-onboard-pipeline.yaml b/.catalog-onboard-pipeline.yaml index 60f32292..6379969f 100644 --- a/.catalog-onboard-pipeline.yaml +++ b/.catalog-onboard-pipeline.yaml @@ -9,5 +9,5 @@ offerings: - name: fully-configurable mark_ready: true install_type: fullstack - pre_validation: "tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh" - post_validation: "tests/scripts/post-validation-destroy-slz-roks-and-logs-instances.sh" + pre_validation: "tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh" + post_validation: "tests/scripts/post-validation-destroy-ocp-and-logs-instances.sh" diff --git a/common-dev-assets b/common-dev-assets index 1f72de03..252e9276 160000 --- a/common-dev-assets +++ b/common-dev-assets @@ -1 +1 @@ -Subproject commit 1f72de03a36f8ee5522aaacf0ba410b6481122e9 +Subproject commit 252e927652827bf9ecb14b950f5bcf3ccd1f69aa diff --git a/tests/pr_test.go b/tests/pr_test.go index 93208fb7..0f1a1f74 100644 --- a/tests/pr_test.go +++ b/tests/pr_test.go @@ -67,10 +67,10 @@ func TestFullyConfigurableSolution(t *testing.T) { var region = validRegions[rand.IntN(len(validRegions))] // ------------------------------------------------------------------------------------------------------ - // Deploy SLZ ROKS Cluster and Observability instances since it is needed to deploy Logs Agent + // Deploy OCP Cluster and Logs instance since it is needed to deploy Logs Agent // ------------------------------------------------------------------------------------------------------ - prefix := fmt.Sprintf("slz-%s", strings.ToLower(random.UniqueId())) + prefix := fmt.Sprintf("ocp-%s", strings.ToLower(random.UniqueId())) realTerraformDir := "./resources" tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId()))) @@ -96,7 +96,7 @@ func TestFullyConfigurableSolution(t *testing.T) { _, existErr := terraform.InitAndApplyE(t, existingTerraformOptions) if existErr != nil { - assert.True(t, existErr == nil, "Init and Apply of temp resources (SLZ-ROKS and Observability Instances) failed") + assert.True(t, existErr == nil, "Init and Apply of temp resources (OCP and Logs Instance) failed") } else { options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ @@ -153,10 +153,10 @@ func TestFullyConfigurableUpgradeSolution(t *testing.T) { var region = validRegions[rand.IntN(len(validRegions))] // ------------------------------------------------------------------------------------------------------ - // Deploy SLZ ROKS Cluster and Observability instances since it is needed to deploy Logs Agent + // Deploy OCP Cluster and Observability instances since it is needed to deploy Logs Agent // ------------------------------------------------------------------------------------------------------ - prefix := fmt.Sprintf("slz-%s", strings.ToLower(random.UniqueId())) + prefix := fmt.Sprintf("ocp-%s", strings.ToLower(random.UniqueId())) realTerraformDir := "./resources" tempTerraformDir, _ := files.CopyTerraformFolderToTemp(realTerraformDir, fmt.Sprintf(prefix+"-%s", strings.ToLower(random.UniqueId()))) @@ -182,7 +182,7 @@ func TestFullyConfigurableUpgradeSolution(t *testing.T) { _, existErr := terraform.InitAndApplyE(t, existingTerraformOptions) if existErr != nil { - assert.True(t, existErr == nil, "Init and Apply of temp resources (SLZ-ROKS and Observability Instances) failed") + assert.True(t, existErr == nil, "Init and Apply of temp resources (OCP and Logs Instances) failed") } else { options := testschematic.TestSchematicOptionsDefault(&testschematic.TestSchematicOptions{ diff --git a/tests/resources/outputs.tf b/tests/resources/outputs.tf index c14e6220..52b34658 100644 --- a/tests/resources/outputs.tf +++ b/tests/resources/outputs.tf @@ -4,7 +4,7 @@ output "region" { value = var.region - description = "Region where SLZ ROKS Cluster is deployed." + description = "Region where OCP Cluster is deployed." } output "prefix" { diff --git a/tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh b/tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh index b9f8fc73..6962d93d 100755 --- a/tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh +++ b/tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh @@ -11,7 +11,7 @@ TF_VARS_FILE="terraform.tfvars" ( cd ${TERRAFORM_SOURCE_DIR} - echo "Destroying prerequisite SLZ OCP Cluster and Logs instance.." + echo "Destroying prerequisite OCP Cluster and Logs instance.." terraform destroy -input=false -auto-approve -var-file=${TF_VARS_FILE} || exit 1 rm -f "${TF_VARS_FILE}" diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh b/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh index c9169036..84a6ef15 100755 --- a/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh +++ b/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh @@ -15,12 +15,12 @@ TF_VARS_FILE="terraform.tfvars" ( cwd=$(pwd) cd ${TERRAFORM_SOURCE_DIR} - echo "Provisioning prerequisite SLZ ROKS CLUSTER and Logs Instance.." + echo "Provisioning prerequisite OCP Cluster and Logs Instance.." terraform init || exit 1 # $VALIDATION_APIKEY is available in the catalog runtime { echo "ibmcloud_api_key=\"${VALIDATION_APIKEY}\"" - echo "prefix=\"slz-$(openssl rand -hex 2)\"" + echo "prefix=\"ocp-$(openssl rand -hex 2)\"" } >> ${TF_VARS_FILE} terraform apply -input=false -auto-approve -var-file=${TF_VARS_FILE} || exit 1 From db1a4031e6d87114d03a5a5eedf44e3a22147ec0 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Thu, 1 May 2025 13:24:25 +0100 Subject: [PATCH 16/21] fix: update code --- ...tances.sh => post-validation-deploy-ocp-and-logs-instances.sh} | 0 ...stances.sh => pre-validation-deploy-ocp-and-logs-instances.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/scripts/{post-validation-deploy-slz-roks-and-logs-instances.sh => post-validation-deploy-ocp-and-logs-instances.sh} (100%) rename tests/scripts/{pre-validation-deploy-slz-roks-and-logs-instances.sh => pre-validation-deploy-ocp-and-logs-instances.sh} (100%) diff --git a/tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh b/tests/scripts/post-validation-deploy-ocp-and-logs-instances.sh similarity index 100% rename from tests/scripts/post-validation-deploy-slz-roks-and-logs-instances.sh rename to tests/scripts/post-validation-deploy-ocp-and-logs-instances.sh diff --git a/tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh b/tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh similarity index 100% rename from tests/scripts/pre-validation-deploy-slz-roks-and-logs-instances.sh rename to tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh From 27b9f9a2745c734392145f3ffe83aab2a561dfd9 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Mon, 5 May 2025 21:40:32 +0100 Subject: [PATCH 17/21] fix: address feedback --- tests/resources/outputs.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/resources/outputs.tf b/tests/resources/outputs.tf index 52b34658..b2e907c0 100644 --- a/tests/resources/outputs.tf +++ b/tests/resources/outputs.tf @@ -7,11 +7,6 @@ output "region" { description = "Region where OCP Cluster is deployed." } -output "prefix" { - value = module.landing_zone.prefix - description = "prefix" -} - output "workload_cluster_id" { value = module.ocp_base.cluster_id description = "ID of the workload cluster." From 9894a1223f6c4d786e9543370462220bb2725520 Mon Sep 17 00:00:00 2001 From: Jordan Date: Mon, 5 May 2025 21:48:02 +0100 Subject: [PATCH 18/21] Update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index dcdf1840..713bd26e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,2 +1,2 @@ # Primary owner should be listed first in list of global owners, followed by any secondary owners -* @Aashiq-J @shemau +* @jor2 @Aashiq-J From 44cb53ac27107703d293a9527b0a513f87736c18 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 6 May 2025 16:08:20 +0100 Subject: [PATCH 19/21] fix: address feedback --- README.md | 2 +- examples/logs-agent-ocp/main.tf | 4 ++-- ibm_catalog.json | 1 - solutions/fully-configurable/DA-types.md | 6 +++--- solutions/fully-configurable/variables.tf | 10 +++++++--- tests/resources/main.tf | 6 +++--- variables.tf | 4 ++-- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e78b7f8c..80729192 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ No modules. | [logs\_agent\_image\_version](#input\_logs\_agent\_image\_version) | The version of the Logs agent image to deploy. | `string` | `"1.5.1"` | no | | [logs\_agent\_log\_source\_namespaces](#input\_logs\_agent\_log\_source\_namespaces) | The list of namespaces from which logs should be forwarded by agent. If namespaces are not listed, logs from all namespaces will be sent. | `list(string)` | `[]` | no | | [logs\_agent\_name](#input\_logs\_agent\_name) | The name of the Logs agent. The name is used in all Kubernetes and Helm resources in the cluster. | `string` | `"logs-agent"` | no | -| [logs\_agent\_namespace](#input\_logs\_agent\_namespace) | The namespace where the Logs agent is deployed. The default value is `ibm-agent`. | `string` | `"ibm-agent"` | no | +| [logs\_agent\_namespace](#input\_logs\_agent\_namespace) | The namespace where the Logs agent is deployed. The default value is `ibm-observe`. | `string` | `"ibm-observe"` | no | | [logs\_agent\_resources](#input\_logs\_agent\_resources) | The resources configuration for cpu/memory/storage. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources) | 
object({
    limits = object({
      cpu    = string
      memory = string
    })
    requests = object({
      cpu    = string
      memory = string
    })
  })
| 
{
  "limits": {
    "cpu": "500m",
    "memory": "3Gi"
  },
  "requests": {
    "cpu": "100m",
    "memory": "1Gi"
  }
}
| no | | [logs\_agent\_selected\_log\_source\_paths](#input\_logs\_agent\_selected\_log\_source\_paths) | The list of specific log sources paths. Logs will only be collected from the specified log source paths. If no paths are specified, it will send logs from `/var/log/containers`. | `list(string)` | `[]` | no | | [logs\_agent\_tolerations](#input\_logs\_agent\_tolerations) | List of tolerations to apply to Logs agent. The default value means a pod will run on every node. | 
list(object({
    key               = optional(string)
    operator          = optional(string)
    value             = optional(string)
    effect            = optional(string)
    tolerationSeconds = optional(number)
  }))
| 
[
  {
    "operator": "Exists"
  }
]
| no | diff --git a/examples/logs-agent-ocp/main.tf b/examples/logs-agent-ocp/main.tf index d3dd4d35..f9494dfc 100644 --- a/examples/logs-agent-ocp/main.tf +++ b/examples/logs-agent-ocp/main.tf @@ -15,7 +15,7 @@ module "resource_group" { ############################################################################## locals { - logs_agent_namespace = "ibm-agent" + logs_agent_namespace = "ibm-observe" logs_agent_name = "logs-agent" } @@ -32,7 +32,7 @@ module "trusted_profile" { service = "logs" }] }] - # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-agent` namespace. + # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-observe` namespace. trusted_profile_links = [{ cr_type = "ROKS_SA" links = [{ diff --git a/ibm_catalog.json b/ibm_catalog.json index 6edf1b56..b5b3389d 100644 --- a/ibm_catalog.json +++ b/ibm_catalog.json @@ -30,7 +30,6 @@ "label": "Fully Configurable", "name": "fully-configurable", "working_directory": "solutions/fully-configurable", - "compliance": {}, "iam_permissions": [ { "service_name": "containers-kubernetes", diff --git a/solutions/fully-configurable/DA-types.md b/solutions/fully-configurable/DA-types.md index c6c4c10b..aec7a41e 100644 --- a/solutions/fully-configurable/DA-types.md +++ b/solutions/fully-configurable/DA-types.md @@ -1,6 +1,6 @@ # Configuring Logs Agent Tolerations -When you deploy the IBM Logs agent using the `terraform-ibm-logs-agent` module, you can configure the tolerations that the agent applies to its pods by using the `logs_agent_tolerations` variable. This variable allows you to specify tolerations for scheduling the logs agent pods on nodes with specific taints. +When you deploy the IBM Logs agent using `Cloud automation for Logs agent`, you can configure the tolerations that the agent applies to its pods by using the `logs_agent_tolerations` variable. This variable allows you to specify tolerations for scheduling the logs agent pods on nodes with specific taints. ### Options for `logs_agent_tolerations` - `key` (optional): The taint key that the toleration applies to. @@ -39,7 +39,7 @@ The `logs_agent_tolerations` variable is used to configure the tolerations for t # Configuring Logs Agent Additional Metadata -When you deploy the IBM logs agent using the `terraform-ibm-logs-agent` module, you can configure additional metadata fields to be added to the routed logs by using the `logs_agent_additional_metadata` variable. This variable allows you to specify key-value pairs of metadata that will be included in the logs. +When you deploy the IBM logs agent using `Cloud automation for Logs agent`, you can configure additional metadata fields to be added to the routed logs by using the `logs_agent_additional_metadata` variable. This variable allows you to specify key-value pairs of metadata that will be included in the logs. ### Options for `logs_agent_additional_metadata` - `key` (optional): The metadata key to add to the logs. @@ -74,7 +74,7 @@ The `logs_agent_additional_metadata` variable is used to configure additional me # Configuring Logs Agent Resources -When you deploy the IBM Logs agent using the `terraform-ibm-logs-agent` module, you can configure the resource requests and limits for the logs agent pods by using the `logs_agent_resources` variable. This variable allows you to specify the CPU and memory resources allocated to the logs agent. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources). +When you deploy the IBM Logs agent using `Cloud automation for Logs agent`, you can configure the resource requests and limits for the logs agent pods by using the `logs_agent_resources` variable. This variable allows you to specify the CPU and memory resources allocated to the logs agent. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources). ### Options for `logs_agent_resources` - `requests` (optional): Specifies the minimum amount of resources required. Includes: diff --git a/solutions/fully-configurable/variables.tf b/solutions/fully-configurable/variables.tf index 0fe7860d..805db54e 100644 --- a/solutions/fully-configurable/variables.tf +++ b/solutions/fully-configurable/variables.tf @@ -21,7 +21,7 @@ variable "cluster_resource_group_id" { variable "cluster_config_endpoint_type" { description = "Specify the type of endpoint to use to access the cluster configuration. Possible values: `default`, `private`, `vpe`, `link`. The `default` value uses the default endpoint of the cluster." type = string - default = "private" + default = "default" nullable = false # use default if null is passed in } @@ -84,8 +84,8 @@ variable "logs_agent_name" { variable "logs_agent_namespace" { type = string - description = "The namespace where the Logs agent is deployed. The default value is `ibm-agent`." - default = "ibm-agent" + description = "The namespace where the Logs agent is deployed. The default value is `ibm-observe`." + default = "ibm-observe" nullable = false } @@ -93,6 +93,10 @@ variable "logs_agent_trusted_profile_id" { type = string description = "The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role. Must provide a value for `logs_agent_iam_api_key` if `logs_agent_trusted_profile_id` is null." default = null + validation { + condition = !(var.logs_agent_iam_mode == "TrustedProfile" && var.logs_agent_trusted_profile_id == null) + error_message = "The `logs_agent_trusted_profile_id` is required when `logs_agent_iam_mode` is set to `TrustedProfile`." + } } variable "logs_agent_iam_api_key" { diff --git a/tests/resources/main.tf b/tests/resources/main.tf index 7b409117..9ea9b0ef 100644 --- a/tests/resources/main.tf +++ b/tests/resources/main.tf @@ -84,7 +84,7 @@ module "ocp_base" { vpc_subnets = local.cluster_vpc_subnets worker_pools = local.worker_pools access_tags = [] - disable_outbound_traffic_protection = true # set as True to enable outbound traffic + disable_outbound_traffic_protection = false } ############################################################################## @@ -165,7 +165,7 @@ module "cloud_logs" { ############################################################################## locals { - logs_agent_namespace = "ibm-agent" + logs_agent_namespace = "ibm-observe" logs_agent_name = "logs-agent" } @@ -181,7 +181,7 @@ module "trusted_profile" { service = "logs" }] }] - # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-agent` namespace. + # Set up fine-grained authorization for `logs-agent` running in ROKS cluster in `ibm-observe` namespace. trusted_profile_links = [{ cr_type = "ROKS_SA" links = [{ diff --git a/variables.tf b/variables.tf index 140273b0..5aa0dc17 100644 --- a/variables.tf +++ b/variables.tf @@ -92,8 +92,8 @@ variable "logs_agent_name" { variable "logs_agent_namespace" { type = string - description = "The namespace where the Logs agent is deployed. The default value is `ibm-agent`." - default = "ibm-agent" + description = "The namespace where the Logs agent is deployed. The default value is `ibm-observe`." + default = "ibm-observe" nullable = false } From 7e3d7d5362ade818625cdbda79588a264798a500 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 6 May 2025 21:09:58 +0100 Subject: [PATCH 20/21] fix: address feedback --- README.md | 4 ++-- examples/logs-agent-ocp/main.tf | 10 +++++----- main.tf | 4 ++-- solutions/fully-configurable/main.tf | 2 +- .../pre-validation-deploy-ocp-and-logs-instances.sh | 12 ++++++------ variables.tf | 6 +++--- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 80729192..2b5525cd 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ module "logs_agent_module" { # update this with the Id of your IBM Cloud resource group cluster_resource_group_id = "resource group id" # Logs Agent variables - logs_agent_trusted_profile = "XXXXXXXX" + logs_agent_trusted_profile_id = "XXXXXXXX" cloud_logs_ingress_endpoint = ".ingress.us-south.logs.cloud.ibm.com" cloud_logs_ingress_port = 443 } @@ -128,7 +128,7 @@ No modules. | [logs\_agent\_resources](#input\_logs\_agent\_resources) | The resources configuration for cpu/memory/storage. [Learn More](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-agent-helm-template-clusters#agent-helm-template-clusters-chart-options-resources) | 
object({
    limits = object({
      cpu    = string
      memory = string
    })
    requests = object({
      cpu    = string
      memory = string
    })
  })
| 
{
  "limits": {
    "cpu": "500m",
    "memory": "3Gi"
  },
  "requests": {
    "cpu": "100m",
    "memory": "1Gi"
  }
}
| no | | [logs\_agent\_selected\_log\_source\_paths](#input\_logs\_agent\_selected\_log\_source\_paths) | The list of specific log sources paths. Logs will only be collected from the specified log source paths. If no paths are specified, it will send logs from `/var/log/containers`. | `list(string)` | `[]` | no | | [logs\_agent\_tolerations](#input\_logs\_agent\_tolerations) | List of tolerations to apply to Logs agent. The default value means a pod will run on every node. | 
list(object({
    key               = optional(string)
    operator          = optional(string)
    value             = optional(string)
    effect            = optional(string)
    tolerationSeconds = optional(number)
  }))
| 
[
  {
    "operator": "Exists"
  }
]
| no | -| [logs\_agent\_trusted\_profile](#input\_logs\_agent\_trusted\_profile) | The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role. | `string` | `null` | no | +| [logs\_agent\_trusted\_profile\_id](#input\_logs\_agent\_trusted\_profile\_id) | The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role. | `string` | `null` | no | | [wait\_till](#input\_wait\_till) | To avoid long wait times when you run your Terraform code, you can specify the stage when you want Terraform to mark the cluster resource creation as completed. Depending on what stage you choose, the cluster creation might not be fully completed and continues to run in the background. However, your Terraform code can continue to run without waiting for the cluster to be fully created. Supported args are `MasterNodeReady`, `OneWorkerNodeReady`, `IngressReady` and `Normal` | `string` | `"Normal"` | no | | [wait\_till\_timeout](#input\_wait\_till\_timeout) | Timeout for wait\_till in minutes. | `number` | `90` | no | diff --git a/examples/logs-agent-ocp/main.tf b/examples/logs-agent-ocp/main.tf index f9494dfc..e2ac676c 100644 --- a/examples/logs-agent-ocp/main.tf +++ b/examples/logs-agent-ocp/main.tf @@ -178,11 +178,11 @@ module "logs_agent" { cluster_id = module.ocp_base.cluster_id cluster_resource_group_id = module.resource_group.resource_group_id # Logs agent - logs_agent_trusted_profile = module.trusted_profile.trusted_profile.id - logs_agent_namespace = local.logs_agent_namespace - logs_agent_name = local.logs_agent_name - cloud_logs_ingress_endpoint = module.cloud_logs.ingress_private_endpoint - cloud_logs_ingress_port = 443 + logs_agent_trusted_profile_id = module.trusted_profile.trusted_profile.id + logs_agent_namespace = local.logs_agent_namespace + logs_agent_name = local.logs_agent_name + cloud_logs_ingress_endpoint = module.cloud_logs.ingress_private_endpoint + cloud_logs_ingress_port = 443 # example of how to add additional metadata to the logs agent logs_agent_additional_metadata = [{ key = "cluster_id" diff --git a/main.tf b/main.tf index 7bf5659a..e18753ed 100644 --- a/main.tf +++ b/main.tf @@ -26,7 +26,7 @@ data "ibm_container_cluster_config" "cluster_config" { locals { logs_agent_selected_log_source_paths = distinct(concat([for namespace in var.logs_agent_log_source_namespaces : "/var/log/containers/*_${namespace}_*.log"], var.logs_agent_selected_log_source_paths)) logs_agent_iam_api_key = var.logs_agent_iam_api_key != null ? var.logs_agent_iam_api_key : "" - logs_agent_trusted_profile = var.logs_agent_trusted_profile != null ? var.logs_agent_trusted_profile : "" + logs_agent_trusted_profile_id = var.logs_agent_trusted_profile_id != null ? var.logs_agent_trusted_profile_id : "" cloud_logs_ingress_endpoint = var.cloud_logs_ingress_endpoint != null ? var.cloud_logs_ingress_endpoint : "" logs_agent_additional_metadata = length(var.logs_agent_additional_metadata) > 0 ? merge([ for metadata in var.logs_agent_additional_metadata : { @@ -74,7 +74,7 @@ resource "helm_release" "logs_agent" { set { name = "env.trustedProfileID" type = "string" - value = local.logs_agent_trusted_profile + value = local.logs_agent_trusted_profile_id } set { name = "env.iamMode" diff --git a/solutions/fully-configurable/main.tf b/solutions/fully-configurable/main.tf index 97323a98..ea0740bd 100644 --- a/solutions/fully-configurable/main.tf +++ b/solutions/fully-configurable/main.tf @@ -26,7 +26,7 @@ module "logs_agent" { logs_agent_image_version = var.logs_agent_image_version logs_agent_name = var.logs_agent_name logs_agent_namespace = var.logs_agent_namespace - logs_agent_trusted_profile = var.logs_agent_trusted_profile_id + logs_agent_trusted_profile_id = var.logs_agent_trusted_profile_id logs_agent_iam_api_key = var.logs_agent_iam_api_key logs_agent_tolerations = var.logs_agent_tolerations logs_agent_additional_log_source_paths = var.logs_agent_additional_log_source_paths diff --git a/tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh b/tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh index 84a6ef15..616d7eba 100755 --- a/tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh +++ b/tests/scripts/pre-validation-deploy-ocp-and-logs-instances.sh @@ -28,23 +28,23 @@ TF_VARS_FILE="terraform.tfvars" cluster_id_value=$(terraform output -state=terraform.tfstate -raw workload_cluster_id) cluster_resource_group_id_var_name="cluster_resource_group_id" cluster_resource_group_id_value=$(terraform output -state=terraform.tfstate -raw cluster_resource_group_id) - logs_agent_trusted_profile_var_name="logs_agent_trusted_profile" - logs_agent_trusted_profile_value=$(terraform output -state=terraform.tfstate -raw trusted_profile_id) + logs_agent_trusted_profile_id_var_name="logs_agent_trusted_profile_id" + logs_agent_trusted_profile_id_value=$(terraform output -state=terraform.tfstate -raw trusted_profile_id) cloud_logs_ingress_endpoint_var_name="cloud_logs_ingress_endpoint" cloud_logs_ingress_endpoint_value=$(terraform output -state=terraform.tfstate -raw cloud_logs_ingress_private_endpoint) - echo "Appending '${cluster_id_var_name}' '${cluster_resource_group_id_var_name}', '${logs_agent_trusted_profile_var_name}', and '${cloud_logs_ingress_endpoint_var_name}' input variable values to ${JSON_FILE}.." + echo "Appending '${cluster_id_var_name}' '${cluster_resource_group_id_var_name}', '${logs_agent_trusted_profile_id_var_name}', and '${cloud_logs_ingress_endpoint_var_name}' input variable values to ${JSON_FILE}.." cd "${cwd}" jq -r --arg cluster_id_var_name "${cluster_id_var_name}" \ --arg cluster_id_value "${cluster_id_value}" \ --arg cluster_resource_group_id_var_name "${cluster_resource_group_id_var_name}" \ --arg cluster_resource_group_id_value "${cluster_resource_group_id_value}" \ - --arg logs_agent_trusted_profile_var_name "${logs_agent_trusted_profile_var_name}" \ - --arg logs_agent_trusted_profile_value "${logs_agent_trusted_profile_value}" \ + --arg logs_agent_trusted_profile_id_var_name "${logs_agent_trusted_profile_id_var_name}" \ + --arg logs_agent_trusted_profile_id_value "${logs_agent_trusted_profile_id_value}" \ --arg cloud_logs_ingress_endpoint_var_name "${cloud_logs_ingress_endpoint_var_name}" \ --arg cloud_logs_ingress_endpoint_value "${cloud_logs_ingress_endpoint_value}" \ - '. + {($cluster_id_var_name): $cluster_id_value, ($cluster_resource_group_id_var_name): $cluster_resource_group_id_value, ($logs_agent_trusted_profile_var_name): $logs_agent_trusted_profile_value, ($cloud_logs_ingress_endpoint_var_name): $cloud_logs_ingress_endpoint_value}' "${JSON_FILE}" > tmpfile && mv tmpfile "${JSON_FILE}" || exit 1 + '. + {($cluster_id_var_name): $cluster_id_value, ($cluster_resource_group_id_var_name): $cluster_resource_group_id_value, ($logs_agent_trusted_profile_id_var_name): $logs_agent_trusted_profile_id_value, ($cloud_logs_ingress_endpoint_var_name): $cloud_logs_ingress_endpoint_value}' "${JSON_FILE}" > tmpfile && mv tmpfile "${JSON_FILE}" || exit 1 echo "Pre-validation complete successfully" ) diff --git a/variables.tf b/variables.tf index 5aa0dc17..656216d0 100644 --- a/variables.tf +++ b/variables.tf @@ -97,13 +97,13 @@ variable "logs_agent_namespace" { nullable = false } -variable "logs_agent_trusted_profile" { +variable "logs_agent_trusted_profile_id" { type = string description = "The IBM Cloud trusted profile ID. Used only when `logs_agent_iam_mode` is set to `TrustedProfile`. The trusted profile must have an IBM Cloud Logs `Sender` role." default = null validation { - condition = !(var.logs_agent_trusted_profile == null && var.logs_agent_iam_mode == "TrustedProfile") - error_message = "The `logs_agent_trusted_profile` is required when `logs_agent_iam_mode` is set to `TrustedProfile`." + condition = !(var.logs_agent_trusted_profile_id == null && var.logs_agent_iam_mode == "TrustedProfile") + error_message = "The `logs_agent_trusted_profile_id` is required when `logs_agent_iam_mode` is set to `TrustedProfile`." } } From 9b4d1a86098bdd0a2b15f316db9206b1b5f87cb9 Mon Sep 17 00:00:00 2001 From: Jordan-Williams2 Date: Tue, 6 May 2025 22:35:11 +0100 Subject: [PATCH 21/21] fix: address feedback --- ...ances.sh => post-validation-destroy-ocp-and-logs-instances.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/scripts/{post-validation-deploy-ocp-and-logs-instances.sh => post-validation-destroy-ocp-and-logs-instances.sh} (100%) diff --git a/tests/scripts/post-validation-deploy-ocp-and-logs-instances.sh b/tests/scripts/post-validation-destroy-ocp-and-logs-instances.sh similarity index 100% rename from tests/scripts/post-validation-deploy-ocp-and-logs-instances.sh rename to tests/scripts/post-validation-destroy-ocp-and-logs-instances.sh