feat: Add the ability to configure cloud logs policies using new input cloud_logs_policies (#209)

Author: maheshwarishikha

solutions/instances/DA-types.md

@@ -1,20 +1,24 @@
-# Configuring Event Notification (EN) Instances for Cloud Logging
+# Configuring complex inputs for Cloud Automation for Observability
 
-An optional input variables in the IBM Cloud [Observability instances deployable architecture](https://cloud.ibm.com/catalog#deployable_architecture) use complex object types. You specify these inputs when you configure deployable architecture.
+Several optional input variables in the IBM Cloud [Observability instances deployable architecture](https://cloud.ibm.com/catalog#deployable_architecture) use complex object types. You specify these inputs when you configure deployable architecture.
+
+* Cloud Logs Event Notification Instances (`cloud_logs_existing_en_instances`)
+* Cloud Logs policies (`cloud_logs_policies`)
 
-- [Cloud Logs Event Notification Instances](#cloud_logs_existing_en_instances) (`cloud_logs_existing_en_instances`)
 
 ## Cloud Logs Event Notification Instances <a name="cloud_logs_existing_en_instances"></a>
 
 The `cloud_logs_existing_en_instances` input variable allows you to provide a list of existing Event Notification (EN) instances that will be integrated with the Cloud Logging service. For each EN instance, you need to specify its CRN (Cloud Resource Name). You can also optionally configure a integration name and control whether to skip the creation of an authentication policy for the instance.
 
 - Variable name: `cloud_logs_existing_en_instances`.
-- Type: A list of objects. Each object represents an EN instance with the following attributes:
+- Type: A list of objects. Each object represents an EN instance.
+- Default value: An empty list (`[]`).
+
+### Options for cloud_logs_existing_en_instances
+
   - `instance_crn` (required): The Cloud Resource Name (CRN) of the Event Notification instance.
   - `integration_name` (optional): The name of the Event Notification integration that gets created. If a prefix input variable is passed, it is prefixed to the value in the `<prefix>-value` format. Defaults to `"cloud-logs-en-integration"`.
   - `skip_en_auth_policy` (optional): A boolean flag to determine whether to skip the creation of an authentication policy that allows Cloud Logs 'Event Source Manager' role access in the existing event notification instance. Defaults to `false`.
-- Default value: An empty list (`[]`).
-
 
 ### Example Event Notification Instance Configuration
 
@@ -35,3 +39,52 @@ cloud_logs_existing_en_instances = [
 In this example:
 - The first EN instance has a integration name `"custom-logging-en-integration"` and skips the authentication policy.
 - The second EN instance uses the default integration name and includes the authentication policy.
+
+## Cloud Logs Policies <a name="cloud_logs_policies"></a>
+
+The `cloud_logs_policies` input variable allows you to provide a list of policies that will be configured in the Cloud Logs service. Refer [here](https://cloud.ibm.com/docs/cloud-logs?topic=cloud-logs-tco-optimizer) for more information.
+
+- Variable name: `cloud_logs_policies`.
+- Type: A list of objects. Each object represents a policy.
+- Default value: An empty list (`[]`).
+
+### Options for cloud_logs_policies
+
+  - `logs_policy_name` (required): The unique policy name.
+  - `logs_policy_description` (optional): The description of the policy to create.
+  - `logs_policy_priority` (required): The priority to determine the pipeline for the logs. Allowed values are: type_unspecified, type_block, type_low, type_medium, type_high. High (priority value) sent to 'Priority insights' (TCO pipleine), Medium to 'Analyze and alert', Low to 'Store and search', Blocked are not sent to any pipeline.
+  - `application_rule` (optional): The rules to include in the policy configuration for matching applications.
+  - `subsystem_rule` (optional): The subsystem rules to include in the policy configuration for matching applications.
+  - `log_rules` (required): The log severities to include in the policy configuration.
+  - `archive_retention` (optional): Define archive retention.
+
+### Example cloud_logs_policies
+
+```hcl
+cloud_logs_policies = [
+  {
+    logs_policy_name     = "logs-policy-1"
+    logs_policy_description = "Send info and debug logs of the application (name starts with `test-system-app`) and the subsytem (name starts with `test-sub-system`) logs to Store nad search pipeline"
+    logs_policy_priority = "type_low"
+    application_rule = [{
+      name         = "test-system-app"
+      rule_type_id = "start_with"
+    }]
+    log_rules = [{
+      severities = ["info", "debug"]
+    }]
+    subsystem_rule = [{
+      name         = "test-sub-system"
+      rule_type_id = "start_with"
+    }]
+  },
+  {
+    logs_policy_name     = "logs-policy-2"
+    logs_policy_description = "Send error logs of all applications and all subsystems to Analyze and Alert pipeline"
+    logs_policy_priority = "type_medium"
+    log_rules = [{
+      severities = ["error"]
+    }]
+  }
+]
+```

solutions/instances/README.md

@@ -13,6 +13,7 @@ This deployable architecture creates observability instances in IBM Cloud and su
 * A KMS-encrypted Object Storage bucket for Cloud Logs metrics, if one is not passed in.
 * An Activity Tracker event route to an Object Storage bucket and Cloud Logs target.
 * An option to integrate Cloud Logs with existing event notification instance.
+* An option to configure Cloud logs policies (TCO Optimizer).
 
 ![observability-instances-deployable-architecture](../../reference-architecture/deployable-architecture-observability-instances.svg)
 

solutions/instances/main.tf

@@ -216,7 +216,7 @@ module "en_crn_parser" {
 module "observability_instance" {
   depends_on        = [time_sleep.wait_for_atracker_cos_authorization_policy]
   source            = "terraform-ibm-modules/observability-instances/ibm"
-  version           = "3.0.2"
+  version           = "3.3.1"
   region            = var.region
   resource_group_id = module.resource_group.resource_group_id
 
@@ -237,6 +237,7 @@ module "observability_instance" {
   cloud_logs_tags              = var.cloud_logs_tags
   cloud_logs_service_endpoints = "public-and-private"
   cloud_logs_retention_period  = var.cloud_logs_retention_period
+  cloud_logs_policies          = var.cloud_logs_policies
   cloud_logs_data_storage = var.cloud_logs_provision ? {
     logs_data = {
       enabled              = true

solutions/instances/outputs.tf

@@ -29,6 +29,12 @@ output "cloud_logs_name" {
   description = "The name of the provisioned Cloud Logs instance."
 }
 
+## Cloud logs policies
+output "logs_policies_details" {
+  value       = length(var.cloud_logs_policies) > 0 ? module.observability_instance.logs_policies_details : null
+  description = "The details of the Cloud logs policies created."
+}
+
 ## Cloud Monitoring
 output "cloud_monitoring_name" {
   value       = var.cloud_monitoring_provision ? module.observability_instance.cloud_monitoring_name : null

solutions/instances/variables.tf

@@ -222,6 +222,34 @@ variable "manage_log_archive_cos_bucket" {
   description = "Log Analysis has been deprecated, and can no longer be deployed wuth this solution, however you can continue to manage the COS bucket that may have been in older versions for Log Analysis log archiving by setting this input to true."
 }
 
+########################################################################
+# Cloud Logs Policies - TCO Optimizer
+#########################################################################
+
+variable "cloud_logs_policies" {
+  type = list(object({
+    logs_policy_name        = string
+    logs_policy_description = optional(string, null)
+    logs_policy_priority    = string
+    application_rule = optional(list(object({
+      name         = string
+      rule_type_id = optional(string, "unspecified")
+    })))
+    subsystem_rule = optional(list(object({
+      name         = string
+      rule_type_id = optional(string, "unspecified")
+    })))
+    log_rules = optional(list(object({
+      severities = list(string)
+    })))
+    archive_retention = optional(list(object({
+      id = string
+    })))
+  }))
+  description = "Configuration of Cloud Logs policies. [Learn more](https://github.com/terraform-ibm-modules/terraform-ibm-observability-da/tree/main/solutions/standard/DA-types.md#cloud_logs_policies)."
+  default     = []
+}
+
 ##############################################################################
 # Activity Tracker Event Routing Variables
 ##############################################################################

tests/pr_test.go

@@ -117,6 +117,17 @@ func TestRunUpgradeSolutionInstances(t *testing.T) {
 		"management_endpoint_type_for_bucket": "public",
 		"enable_platform_logs":                "false",
 		"enable_platform_metrics":             "false",
+		"cloud_logs_policies": []map[string]interface{}{
+			{
+				"logs_policy_name":     "upg-test-policy",
+				"logs_policy_priority": "type_low",
+				"log_rules": []map[string]interface{}{
+					{
+						"severities": []string{"info", "debug"},
+					},
+				},
+			},
+		},
 	}
 
 	output, err := options.RunTestUpgrade()
@@ -277,6 +288,17 @@ func TestRunExistingResourcesInstances(t *testing.T) {
 				"management_endpoint_type_for_bucket": "public",
 				"enable_platform_metrics":             "false",
 				"enable_platform_logs":                "false",
+				"cloud_logs_policies": []map[string]interface{}{
+					{
+						"logs_policy_name":     "test-policy",
+						"logs_policy_priority": "type_low",
+						"log_rules": []map[string]interface{}{
+							{
+								"severities": []string{"info"},
+							},
+						},
+					},
+				},
 			},
 		})