feat: common namespace

ExpediaGroup · Nov 12, 2024 · 9e5c24b · 9e5c24b
1 parent d20ace2
commit 9e5c24b
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 35 deletions.
diff --git a/templates/apiary-hms-readonly.json b/templates/apiary-hms-readonly.json
@@ -75,7 +75,7 @@
         }
     },
     "dockerLabels": {
-        "com.datadoghq.ad.instances": "[{ \"prometheus_url\": \"http://%%host%%:8080/actuator/prometheus\", \"namespace\": \"${hms_readonly_namespace}\", \"metrics\": [\"metrics_classloading_loaded_value\", \"metrics_threads_count_value\", \"metrics_memory_heap_max_value\", \"metrics_init_total_count_tables_value\", \"metrics_init_total_count_dbs_value\", \"metrics_memory_heap_used_value\", \"metrics_init_total_count_partitions_value\", \"jvm_threads_current\", \"jvm_threads_started_total\", \"jvm_memory_bytes_used\", \"jvm_memory_bytes_init\", \"jvm_gc_collection_seconds_count\", \"jvm_gc_collection_seconds\", \"process_cpu_seconds_total\", \"java_lang_operatingsystem_processcpuload\", \"java_lang_operatingsystem_processcputime\", \"metrics_threads_runnable_count_value\", \"metrics_threads_waiting_count_value\", \"java_lang_memory_heapmemoryusage_used\", \"metrics_memory_heap_init_value\", \"metrics_api_get_partition_by_name_count\", \"metrics_api_get_partitions_by_names_count\", \"metrics_api_get_partition_names_count\", \"metrics_api_get_partitions_by_expr_count\", \"metrics_api_get_partitions_count\", \"metrics_api_get_partition_count\", \"metrics_api_get_partitions_by_filter_count\", \"metrics_api_add_partitions_count\", \"metrics_api_add_partitions_req_count\", \"metrics_api_drop_partition_by_name_count\", \"metrics_api_add_partition_count\", \"metrics_api_alter_partitions_count\", \"metrics_api_create_table_count\", \"metrics_api_alter_table_with_cascade_count\", \"metrics_api_get_table_meta_count\", \"metrics_api_get_table_metas_count\", \"metrics_api_get_table_count\", \"metrics_api_alter_table_count\", \"metrics_api_get_tables_count\", \"metrics_api_get_all_tables_count\", \"metrics_api_drop_table_count\", \"metrics_api_get_multi_table_count\", \"metrics_api_get_database_count\", \"metrics_api_get_all_databases_count\", \"metrics_api_get_databases_count\", \"metrics_api_create_function_count\", \"metrics_api_getmetaconf_count\", \"metrics_api_alter_table_with_environment_context_count\", \"metrics_api_delete_column_statistics_by_table_count\", \"metrics_api_get_functions_count\", \"metrics_api_get_function_count\", \"metrics_api_shutdown_count\", \"metrics_api_flushcache_count\", \"metrics_api_get_indexes_count\", \"metrics_api_get_config_value_count\", \"metrics_api_set_ugi_count\", \"metrics_api_get_all_functions_count\", \"metrics_api_get_table_req_95thpercentile\", \"metrics_api_get_table_req_50thpercentile\", \"metrics_api_get_table_req_count\", \"metrics_api_get_table_req_max\", \"metrics_api_get_databases_count\", \"metrics_api_get_databases_95thpercentile\", \"metrics_api_get_databases_50thpercentile\", \"metrics_api_get_databases_max\", \"metrics_api_get_partitions_95thpercentile\", \"metrics_api_get_partitions_50thpercentile\", \"metrics_api_get_partitions_count\", \"metrics_api_get_partitions_max\", \"metrics_api_get_partitions_50thpercentile\", \"metrics_api_get_table_req_50thpercentile\", \"metrics_api_get_database_95thpercentile\", \"metrics_api_get_database_50thpercentile\", \"metrics_kafka_listener_failures_count\", \"metrics_kafka_listener_successes_count\", \"metrics_api_get_table_objects_by_name_req_max\" ], \"type_overrides\": { \"metrics_classloading_loaded_value\": \"gauge\", \"metrics_threads_count_value\": \"gauge\", \"metrics_memory_heap_max_value\": \"gauge\", \"metrics_init_total_count_tables_value\": \"gauge\", \"metrics_init_total_count_dbs_value\": \"gauge\", \"metrics_memory_heap_used_value\": \"gauge\", \"metrics_init_total_count_partitions_value\": \"gauge\", \"jvm_threads_current\": \"gauge\", \"jvm_threads_started_total\": \"gauge\", \"jvm_memory_bytes_used\": \"gauge\", \"jvm_memory_bytes_init\": \"gauge\", \"jvm_gc_collection_seconds_count\": \"gauge\", \"jvm_gc_collection_seconds\": \"gauge\", \"process_cpu_seconds_total\": \"gauge\", \"java_lang_operatingsystem_processcpuload\": \"gauge\", \"java_lang_operatingsystem_processcputime\": \"gauge\", \"metrics_threads_runnable_count_value\": \"gauge\", \"metrics_threads_waiting_count_value\": \"gauge\", \"java_lang_memory_heapmemoryusage_used\": \"gauge\", \"metrics_memory_heap_init_value\": \"gauge\", \"metrics_api_get_partition_by_name_count\": \"gauge\", \"metrics_api_get_partitions_by_names_count\": \"gauge\", \"metrics_api_get_partition_names_count\": \"gauge\", \"metrics_api_get_partitions_by_expr_count\": \"gauge\", \"metrics_api_get_partitions_count\": \"gauge\", \"metrics_api_get_partition_count\": \"gauge\", \"metrics_api_get_partitions_by_filter_count\": \"gauge\", \"metrics_api_add_partitions_count\": \"gauge\", \"metrics_api_add_partitions_req_count\": \"gauge\", \"metrics_api_drop_partition_by_name_count\": \"gauge\", \"metrics_api_add_partition_count\": \"gauge\", \"metrics_api_alter_partitions_count\": \"gauge\", \"metrics_api_create_table_count\": \"gauge\", \"metrics_api_alter_table_with_cascade_count\": \"gauge\", \"metrics_api_get_table_meta_count\": \"gauge\", \"metrics_api_get_table_metas_count\": \"gauge\", \"metrics_api_get_table_count\": \"gauge\", \"metrics_api_alter_table_count\": \"gauge\", \"metrics_api_get_tables_count\": \"gauge\", \"metrics_api_get_all_tables_count\": \"gauge\", \"metrics_api_drop_table_count\": \"gauge\", \"metrics_api_get_multi_table_count\": \"gauge\", \"metrics_api_get_database_count\": \"gauge\", \"metrics_api_get_all_databases_count\": \"gauge\", \"metrics_api_get_databases_count\": \"gauge\", \"metrics_api_create_function_count\": \"gauge\", \"metrics_api_getmetaconf_count\": \"gauge\", \"metrics_api_alter_table_with_environment_context_count\": \"gauge\", \"metrics_api_delete_column_statistics_by_table_count\": \"gauge\", \"metrics_api_get_functions_count\": \"gauge\", \"metrics_api_get_function_count\": \"gauge\", \"metrics_api_shutdown_count\": \"gauge\", \"metrics_api_flushcache_count\": \"gauge\", \"metrics_api_get_indexes_count\": \"gauge\", \"metrics_api_get_config_value_count\": \"gauge\", \"metrics_api_set_ugi_count\": \"gauge\", \"metrics_api_get_all_functions_count\": \"gauge\", \"metrics_api_get_table_req_95thpercentile\": \"gauge\", \"metrics_api_get_table_req_50thpercentile\": \"gauge\", \"metrics_api_get_table_req_count\": \"gauge\", \"metrics_api_get_table_req_max\": \"gauge\", \"metrics_api_get_databases_count\": \"gauge\", \"metrics_api_get_databases_95thpercentile\": \"gauge\", \"metrics_api_get_databases_50thpercentile\": \"gauge\", \"metrics_api_get_databases_max\": \"gauge\", \"metrics_api_get_partitions_95thpercentile\": \"gauge\", \"metrics_api_get_partitions_50thpercentile\": \"gauge\", \"metrics_api_get_partitions_count\": \"gauge\", \"metrics_api_get_partitions_max\": \"gauge\", \"metrics_api_get_partitions_50thpercentile\": \"gauge\", \"metrics_api_get_table_req_50thpercentile\": \"gauge\",\"metrics_api_get_database_95thpercentile\": \"gauge\",\"metrics_api_get_database_50thpercentile\": \"gauge\", \"metrics_kafka_listener_failures_count\": \"gauge\", \"metrics_kafka_listener_successes_count\": \"gauge\", \"metrics_api_get_table_objects_by_name_req_max\": \"gauge\"} }]",
+        "com.datadoghq.ad.instances": "[{ \"prometheus_url\": \"http://%%host%%:8080/actuator/prometheus\", \"namespace\": \"${hms_readonly_namespace}\", \"metrics\": [ \"${join("\",\"", "${var.datadog_metrics_hms_readwrite_readonly} )}\" ] }]",
         "com.datadoghq.ad.check_names": "[\"prometheus\"]",
         "com.datadoghq.ad.init_configs": "[{}]"
     },

diff --git a/variables.tf b/variables.tf
@@ -545,30 +545,6 @@ EOF
   default = []
 }
 
-variable "hms_ro_tolerations" {
-  description = <<EOF
-Adds a list of tolerations for the HMS readonly pods. For example if you
-have a pool of workers with the following taints "pool=metastore:NoSchedule" you
-can add a toleration like this:
-
-hms_rw_tolerations = [
-  {
-    key      = "pool"
-    operator = "Equal"
-    value    = "metastore"
-    effect   = "NoSchedule"
-  }
-]
-EOF  
-  type = list(object({
-    effect             = string
-    key                = string
-    operator           = string
-    value              = string
-  }))
-  default = []
-}
-
 variable "enable_autoscaling" {
   description = "Enable read only Hive Metastore k8s horizontal pod autoscaling"
   type        = bool
@@ -892,17 +868,9 @@ variable "hms_housekeeper_additional_environment_variables" {
 }
 
 variable "datadog_metrics_hms_readwrite_readonly" {
-  description = "HMS metrics to be sent to Datadog."
+  description = "HMS metrics to be sent to Datadog for both Readonly and Readwrite"
   type        = list(string)
-  default = [
-    "metrics_classloading_loaded_value",
-    "metrics_threads_count_value",
-    "metrics_memory_heap_max_value",
-    "metrics_init_total_count_tables_value",
-    "metrics_init_total_count_dbs_value",
-    "metrics_memory_heap_used_value",
-    "metrics_init_total_count_partitions_value"
-  ]
+  default     = ["metrics_classloading_loaded_value", "metrics_threads_count_value", "metrics_memory_heap_max_value", "metrics_init_total_count_tables_value", "metrics_init_total_count_dbs_value", "metrics_memory_heap_used_value", "metrics_init_total_count_partitions_value", "jvm_threads_current", "jvm_threads_started_total", "jvm_memory_bytes_used", "jvm_memory_bytes_init", "jvm_gc_collection_seconds_count", "jvm_gc_collection_seconds", "process_cpu_seconds_total", "java_lang_operatingsystem_processcpuload", "java_lang_operatingsystem_processcputime", "metrics_threads_runnable_count_value", "metrics_threads_waiting_count_value", "java_lang_memory_heapmemoryusage_used", "metrics_memory_heap_init_value", "metrics_api_get_partition_by_name_count", "metrics_api_get_partitions_by_names_count", "metrics_api_get_partition_names_count", "metrics_api_get_partitions_by_expr_count", "metrics_api_get_partitions_count", "metrics_api_get_partition_count", "metrics_api_get_partitions_by_filter_count",  "metrics_api_get_partitions_by_filter_50thpercentile", "metrics_api_get_partitions_by_filter_95thpercentile", "metrics_api_get_partitions_by_filter_999thpercentile", "metrics_api_add_partitions_count", "metrics_api_add_partitions_req_count", "metrics_api_drop_partition_by_name_count", "metrics_api_add_partition_count", "metrics_api_alter_partitions_count", "metrics_api_create_table_count", "metrics_api_alter_table_with_cascade_count", "metrics_api_get_table_meta_count", "metrics_api_get_table_metas_count", "metrics_api_get_table_count", "metrics_api_alter_table_count", "metrics_api_get_tables_count", "metrics_api_get_all_tables_count", "metrics_api_drop_table_count", "metrics_api_get_multi_table_count", "metrics_api_get_database_count", "metrics_api_get_all_databases_count", "metrics_api_get_databases_count", "metrics_api_create_function_count", "metrics_api_getmetaconf_count", "metrics_api_alter_table_with_environment_context_count", "metrics_api_delete_column_statistics_by_table_count", "metrics_api_get_functions_count", "metrics_api_get_function_count", "metrics_api_shutdown_count", "metrics_api_flushcache_count", "metrics_api_get_indexes_count", "metrics_api_get_config_value_count", "metrics_api_set_ugi_count", "metrics_api_get_all_functions_count", "metrics_api_get_table_req_50thpercentile", "metrics_api_get_table_req_95thpercentile", "metrics_api_get_table_req_999thpercentile", "metrics_api_get_table_req_count", "metrics_api_get_table_req_max", "metrics_api_get_databases_count", "metrics_api_get_databases_50thpercentile", "metrics_api_get_databases_95thpercentile", "metrics_api_get_databases_999thpercentile", "metrics_api_get_databases_max", "metrics_api_get_partitions_50thpercentile", "metrics_api_get_partitions_95thpercentile", "metrics_api_get_partitions_999thpercentile", "metrics_api_get_partitions_count", "metrics_api_get_partitions_max", "metrics_api_get_partitions_50thpercentile", "metrics_api_get_partitions_95thpercentile", "metrics_api_get_partitions_999thpercentile", "metrics_api_get_database_50thpercentile", "metrics_api_get_database_95thpercentile", "metrics_api_get_database_999thpercentile", "metrics_kafka_listener_failures_count", "metrics_kafka_listener_successes_count", "metrics_api_get_table_objects_by_name_req_max", "metrics_open_connections_count", "java_lang_memory_heapmemoryusage_max", "metrics_memory_non_heap_used_value", "metrics_memory_non_heap_max_value", "java_lang_garbagecollector_lastgcinfo_duration", "metrics_jvm_pause_warn_threshold_count", "metrics_jvm_pause_extrasleeptime_count", "metrics_threads_deadlock_count_value"]
 }
 
 variable "datadog_metrics_enabled" {
@@ -1025,4 +993,26 @@ variable "ecs_requires_compatibilities" {
   default     = ["EC2", "FARGATE"]
 }
 
+variable "hms_ro_tolerations" {
+  description = <<EOF
+  Adds a list of tolerations for the HMS readonly pods. For example if you
+have a pool of workers with the following taints "pool=metastore:NoSchedule" you
+can add a toleration like this:
 
+hms_rw_tolerations = [
+  {
+    key      = "pool"
+    operator = "Equal"
+    value    = "metastore"
+    effect   = "NoSchedule"
+  }
+]
+EOF
+  type = list(object({
+    effect             = string
+    key                = string
+    operator           = string
+    value              = string
+  }))
+  default = []
+}