Skip to content

Commit

Permalink
feat: armonik input modules gcp (#1312)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lemaitre-aneo
lemaitre-aneo authored Aug 20, 2024
2 parents 037fd73 + 2dcd949 commit fb0a0bb
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 112 deletions.
24 changes: 9 additions & 15 deletions infrastructure/quick-deploy/gcp/armonik.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,17 @@ module "armonik" {
source = "./generated/infra-modules/armonik"
namespace = local.namespace
logging_level = var.logging_level
extra_conf = merge(var.extra_conf, {
core = merge(var.extra_conf.core, { PubSub__ProjectId = data.google_client_config.current.project, PubSub__KmsKeyName = data.google_kms_crypto_key.kms.id })

configurations = merge(var.configurations, {
core = [module.pubsub, module.memorystore, module.mongodb, var.configurations.core]
})
jobs_in_database_extra_conf = var.jobs_in_database_extra_conf

// To avoid the "known after apply" behavior that arises from using depends_on, we are using a ternary expression to impose implicit dependencies on the below secrets.
fluent_bit_secret_name = kubernetes_secret.fluent_bit.id != null ? kubernetes_secret.fluent_bit.metadata[0].name : kubernetes_secret.fluent_bit.metadata[0].name
grafana_secret_name = kubernetes_secret.grafana.id != null ? kubernetes_secret.grafana.metadata[0].name : kubernetes_secret.grafana.metadata[0].name
prometheus_secret_name = kubernetes_secret.prometheus.id != null ? kubernetes_secret.prometheus.metadata[0].name : kubernetes_secret.prometheus.metadata[0].name
metrics_exporter_secret_name = kubernetes_secret.metrics_exporter.id != null ? kubernetes_secret.metrics_exporter.metadata[0].name : kubernetes_secret.metrics_exporter.metadata[0].name
partition_metrics_exporter_secret_name = kubernetes_secret.partition_metrics_exporter.id != null ? kubernetes_secret.partition_metrics_exporter.metadata[0].name : kubernetes_secret.partition_metrics_exporter.metadata[0].name
seq_secret_name = kubernetes_secret.seq.id != null ? kubernetes_secret.seq.metadata[0].name : kubernetes_secret.seq.metadata[0].name
shared_storage_secret_name = kubernetes_secret.shared_storage.id != null ? kubernetes_secret.shared_storage.metadata[0].name : kubernetes_secret.shared_storage.metadata[0].name
deployed_object_storage_secret_name = kubernetes_secret.deployed_object_storage.id != null ? kubernetes_secret.deployed_object_storage.metadata[0].name : kubernetes_secret.deployed_object_storage.metadata[0].name
deployed_table_storage_secret_name = kubernetes_secret.deployed_table_storage.id != null ? kubernetes_secret.deployed_table_storage.metadata[0].name : kubernetes_secret.deployed_table_storage.metadata[0].name
deployed_queue_storage_secret_name = kubernetes_secret.deployed_queue_storage.id != null ? kubernetes_secret.deployed_queue_storage.metadata[0].name : kubernetes_secret.deployed_queue_storage.metadata[0].name
s3_secret_name = can(coalesce(kubernetes_secret.gcs[0].id)) ? kubernetes_secret.gcs[0].metadata[0].name : ""
fluent_bit = module.fluent_bit
grafana = one(module.grafana)
prometheus = module.prometheus
metrics = module.metrics_exporter
seq = one(module.seq)
shared_storage_settings = local.shared_storage

// If compute plane has no partition data, provides a default
// but always overrides the images
Expand Down
34 changes: 20 additions & 14 deletions infrastructure/quick-deploy/gcp/parameters.tfvars
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,11 +287,13 @@ metrics_exporter = {
#parition_metrics_exporter = {
# node_selector = { service = "metrics" }
# extra_conf = {
# env = {
# MongoDB__AllowInsecureTls = true
# Serilog__MinimumLevel = "Information"
# MongoDB__TableStorage__PollingDelayMin = "00:00:01"
# MongoDB__TableStorage__PollingDelayMax = "00:00:10"
# MongoDB__DataRetention = "1.00:00:00" # 1 day retention
#}
# }
#}

Expand Down Expand Up @@ -554,28 +556,32 @@ authentication = {
node_selector = { service = "control-plane" }
}

extra_conf = {
configurations = {
core = {
MongoDB__TableStorage__PollingDelayMin = "00:00:01"
MongoDB__TableStorage__PollingDelayMax = "00:00:10"
MongoDB__TableStorage__PollingDelay = "00:00:01"
MongoDB__AllowInsecureTls = true
MongoDB__DataRetention = "1.00:00:00" # 1 day retention
Redis__Timeout = 3000
Redis__SslHost = ""
Redis__TtlTimeSpan = "1.00:00:00" # 1 day retention
env = {
MongoDB__TableStorage__PollingDelayMin = "00:00:01"
MongoDB__TableStorage__PollingDelayMax = "00:00:10"
MongoDB__TableStorage__PollingDelay = "00:00:01"
MongoDB__AllowInsecureTls = true
MongoDB__DataRetention = "1.00:00:00" # 1 day retention
Redis__Timeout = 3000
Redis__SslHost = ""
Redis__TtlTimeSpan = "1.00:00:00" # 1 day retention
}
}
control = {
Submitter__MaxErrorAllowed = 50
env = {
Submitter__MaxErrorAllowed = 50
}
}
worker = {
target_zip_path = "/tmp"
env = {
target_zip_path = "/tmp"
}
}
jobs = { env = { MongoDB__DataRetention = "1.00:00:00" } }
}

# Extra configuration for jobs connecting to database
jobs_in_database_extra_conf = { MongoDB__DataRetention = "1.00:00:00" }

environment_description = {
name = "gcp-dev"
version = "0.0.0"
Expand Down
81 changes: 15 additions & 66 deletions infrastructure/quick-deploy/gcp/storage.tf
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,14 @@
locals {
region = coalesce(var.region, data.google_client_config.current.region)
storage_endpoint_url = {
table_storage_adapter = "MongoDB"
deployed_table_storages = ["MongoDB"]
mongodb = {
url = module.mongodb.url
number_of_replicas = var.mongodb.replicas
}
queue_storage_adapter = "PubSub"
deployed_queue_storages = ["PubSub"]
deployed_object_storages = concat(
length(module.gcs_os) > 0 ? ["S3"] : [],
length(module.memorystore) > 0 ? ["Redis"] : [],
)
object_storage_adapter = try(coalesce(
length(module.gcs_os) > 0 ? "S3" : null,
length(module.memorystore) > 0 ? "Redis" : null,
), "")
redis = length(module.memorystore) > 0 ? {
url = module.memorystore[0].url
} : null
s3 = length(module.gcs_os) > 0 ? {
url = "https://storage.googleapis.com"
bucket_name = module.gcs_os[0].name
kms_key_id = local.kms_key_id
} : null
shared = {
service_url = "https://storage.googleapis.com"
name = module.gcs_fs.name
kms_key_id = local.kms_key_id
}
shared_storage = {
file_storage_type = "S3"
service_url = "https://storage.googleapis.com"
access_key_id = google_storage_hmac_key.cloud_storage.access_id
secret_access_key = google_storage_hmac_key.cloud_storage.secret
name = module.gcs_fs.name
must_force_path_style = false
use_chunk_encoding = false
use_check_sum = false
}
}

Expand All @@ -49,31 +28,6 @@ module "mongodb" {
persistent_volume = null
}

resource "kubernetes_secret" "deployed_table_storage" {
metadata {
name = "deployed-table-storage"
namespace = local.namespace
}
data = {
list = join(",", local.storage_endpoint_url.deployed_table_storages)
adapter = local.storage_endpoint_url.table_storage_adapter
}
}

# PubSub for task queues
resource "kubernetes_secret" "deployed_queue_storage" {
metadata {
name = "deployed-queue-storage"
namespace = local.namespace
}
data = {
list = join(",", local.storage_endpoint_url.deployed_queue_storages)
adapter = local.storage_endpoint_url.queue_storage_adapter
adapter_class_name = "ArmoniK.Core.Adapters.PubSub.QueueBuilder"
adapter_absolute_path = "/adapters/queue/pubsub/ArmoniK.Core.Adapters.PubSub.dll"
}
}

# Redis for payloads
module "memorystore" {
count = var.memorystore != null ? 1 : 0
Expand Down Expand Up @@ -103,17 +57,6 @@ module "memorystore" {
depends_on = [module.psa]
}

resource "kubernetes_secret" "deployed_object_storage" {
metadata {
name = "deployed-object-storage"
namespace = local.namespace
}
data = {
list = join(",", local.storage_endpoint_url.deployed_object_storages)
adapter = local.storage_endpoint_url.object_storage_adapter
}
}

resource "kubernetes_secret" "memorystore" {
count = length(module.memorystore) > 0 ? 1 : 0
metadata {
Expand Down Expand Up @@ -214,3 +157,9 @@ resource "kubernetes_secret" "gcs" {
use_check_sum = false
}
}

module "pubsub" {
source = "./generated/infra-modules/storage/gcp/pubsub"
project_id = data.google_client_config.current.project
kms_key_id = data.google_kms_crypto_key.kms.id
}
27 changes: 10 additions & 17 deletions infrastructure/quick-deploy/gcp/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,25 +292,18 @@ variable "fluent_bit" {
}

# Extra configuration
variable "extra_conf" {
description = "Add extra configuration in the configmaps"
variable "configurations" {
description = ""
type = object({
compute = optional(map(string), {})
control = optional(map(string), {})
core = optional(map(string), {})
log = optional(map(string), {})
metrics = optional(map(string), {})
polling = optional(map(string), {})
worker = optional(map(string), {})
core = optional(any, [])
control = optional(any, [])
compute = optional(any, [])
worker = optional(any, [])
polling = optional(any, [])
log = optional(any, [])
metrics = optional(any, [])
jobs = optional(any, [])
})
default = {}
}

# Extra configuration for jobs connecting to database
variable "jobs_in_database_extra_conf" {
description = "Add extra configuration in the configmaps for jobs connecting to database"
type = map(string)
default = {}
}

# Job to insert partitions in the database
Expand Down

0 comments on commit fb0a0bb

Please sign in to comment.