From 440cca0cd391d51fb5ac42c93066f8fdc168c305 Mon Sep 17 00:00:00 2001 From: tschneider-aneo Date: Tue, 21 May 2024 10:32:34 +0200 Subject: [PATCH] refactor: adapt quick-deploy for refactored mongodb module Update MongoDB connections scripts tools --- infrastructure/quick-deploy/aws/storage.tf | 14 +-- infrastructure/quick-deploy/aws/variables.tf | 14 +-- infrastructure/quick-deploy/gcp/storage.tf | 17 ++-- infrastructure/quick-deploy/gcp/variables.tf | 12 +-- .../quick-deploy/localhost/storage.tf | 16 ++-- .../quick-deploy/localhost/variables.tf | 12 +-- tools/access-mongo-as-admin.sh | 8 -- tools/access-mongo-as-user.sh | 8 -- .../access-mongo-from-kubernetes-as-admin.sh | 2 +- tools/access-mongo-from-kubernetes-as-user.sh | 88 +++++++++++++++++++ versions.tfvars.json | 8 +- 11 files changed, 145 insertions(+), 54 deletions(-) delete mode 100755 tools/access-mongo-as-admin.sh delete mode 100755 tools/access-mongo-as-user.sh create mode 100755 tools/access-mongo-from-kubernetes-as-user.sh diff --git a/infrastructure/quick-deploy/aws/storage.tf b/infrastructure/quick-deploy/aws/storage.tf index a9a9da968..ed292c998 100644 --- a/infrastructure/quick-deploy/aws/storage.tf +++ b/infrastructure/quick-deploy/aws/storage.tf @@ -166,11 +166,13 @@ module "mongodb" { source = "./generated/infra-modules/storage/onpremise/mongodb" namespace = local.namespace mongodb = { - image = local.ecr_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].name - tag = local.ecr_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].tag - node_selector = var.mongodb.node_selector - image_pull_secrets = var.mongodb.pull_secrets - replicas_number = var.mongodb.replicas_number + image = local.ecr_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].name + tag = local.ecr_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].tag + node_selector = var.mongodb.node_selector + image_pull_secrets = var.mongodb.pull_secrets + replicas = var.mongodb.replicas + helm_chart_repository = try(coalesce(var.mongodb.helm_chart_repository), var.helm_charts.mongodb.repository) + helm_chart_version = try(coalesce(var.mongodb.helm_chart_version), var.helm_charts.mongodb.version) } persistent_volume = var.mongodb.persistent_volume != null ? { @@ -359,7 +361,7 @@ locals { } : null mongodb = { url = module.mongodb.url - number_of_replicas = var.mongodb.replicas_number + number_of_replicas = var.mongodb.replicas } shared = { service_url = "https://s3.${var.region}.amazonaws.com" diff --git a/infrastructure/quick-deploy/aws/variables.tf b/infrastructure/quick-deploy/aws/variables.tf index d38d9d743..b5be5af9c 100644 --- a/infrastructure/quick-deploy/aws/variables.tf +++ b/infrastructure/quick-deploy/aws/variables.tf @@ -315,11 +315,14 @@ variable "mq_credentials" { variable "mongodb" { description = "Parameters of MongoDB" type = object({ - image_name = optional(string, "mongo") - image_tag = optional(string) - node_selector = optional(any, {}) - pull_secrets = optional(string, "") - replicas_number = optional(number, 1) + image_name = optional(string, "bitnami/mongodb") + image_tag = optional(string) + node_selector = optional(any, {}) + pull_secrets = optional(string, "") + replicas = optional(number, 1) + helm_chart_repository = optional(string) + helm_chart_version = optional(string) + persistent_volume = optional(object({ storage_provisioner = string volume_binding_mode = optional(string, "Immediate") @@ -334,6 +337,7 @@ variable "mongodb" { })) }), {}) })) + security_context = optional(object({ run_as_user = optional(number, 999) fs_group = optional(number, 999) diff --git a/infrastructure/quick-deploy/gcp/storage.tf b/infrastructure/quick-deploy/gcp/storage.tf index d47cdb07a..f6ba25a41 100644 --- a/infrastructure/quick-deploy/gcp/storage.tf +++ b/infrastructure/quick-deploy/gcp/storage.tf @@ -5,7 +5,7 @@ locals { deployed_table_storages = ["MongoDB"] mongodb = { url = module.mongodb.url - number_of_replicas = var.mongodb.replicas_number + number_of_replicas = var.mongodb.replicas } queue_storage_adapter = "PubSub" deployed_queue_storages = ["PubSub"] @@ -38,11 +38,13 @@ module "mongodb" { source = "./generated/infra-modules/storage/onpremise/mongodb" namespace = local.namespace mongodb = { - image = local.docker_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].name - tag = local.docker_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].tag - node_selector = var.mongodb.node_selector - image_pull_secrets = var.mongodb.pull_secrets - replicas_number = var.mongodb.replicas_number + image = local.docker_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].name + tag = local.docker_images["${var.mongodb.image_name}:${try(coalesce(var.mongodb.image_tag), "")}"].tag + node_selector = var.mongodb.node_selector + image_pull_secrets = var.mongodb.pull_secrets + replicas = var.mongodb.replicas + helm_chart_repository = try(coalesce(var.mongodb.helm_chart_repository), var.helm_charts.mongodb.repository) + helm_chart_version = try(coalesce(var.mongodb.helm_chart_version), var.helm_charts.mongodb.version) } persistent_volume = null } @@ -77,6 +79,7 @@ module "memorystore" { count = var.memorystore != null ? 1 : 0 source = "./generated/infra-modules/storage/gcp/memorystore/redis" name = "${local.prefix}-redis" + namespace = local.namespace memory_size_gb = var.memorystore.memory_size_gb auth_enabled = var.memorystore.auth_enabled authorized_network = module.vpc.name @@ -158,6 +161,7 @@ resource "google_project_iam_member" "allow_gcs_access" { module "gcs_fs" { source = "./generated/infra-modules/storage/gcp/gcs" name = "${local.prefix}-gcsfs" + namespace = local.namespace location = local.region default_kms_key_name = local.kms_key_id force_destroy = true @@ -187,6 +191,7 @@ module "gcs_os" { count = var.gcs_os != null ? 1 : 0 source = "./generated/infra-modules/storage/gcp/gcs" name = "${local.prefix}-gcsos" + namespace = local.namespace location = local.region default_kms_key_name = local.kms_key_id force_destroy = true diff --git a/infrastructure/quick-deploy/gcp/variables.tf b/infrastructure/quick-deploy/gcp/variables.tf index af084b2ed..0d487f1ad 100644 --- a/infrastructure/quick-deploy/gcp/variables.tf +++ b/infrastructure/quick-deploy/gcp/variables.tf @@ -111,11 +111,13 @@ variable "chaos_mesh" { variable "mongodb" { description = "Parameters of MongoDB" type = object({ - image_name = optional(string, "mongo") - image_tag = optional(string) - node_selector = optional(any, {}) - pull_secrets = optional(string, "") - replicas_number = optional(number, 1) + image_name = optional(string, "bitnami/mongodb") + image_tag = optional(string) + node_selector = optional(any, {}) + pull_secrets = optional(string, "") + replicas = optional(number, 1) + helm_chart_repository = optional(string) + helm_chart_version = optional(string) }) default = {} } diff --git a/infrastructure/quick-deploy/localhost/storage.tf b/infrastructure/quick-deploy/localhost/storage.tf index 820fab650..06ba2dc0b 100644 --- a/infrastructure/quick-deploy/localhost/storage.tf +++ b/infrastructure/quick-deploy/localhost/storage.tf @@ -26,11 +26,13 @@ module "mongodb" { source = "./generated/infra-modules/storage/onpremise/mongodb" namespace = local.namespace mongodb = { - image = var.mongodb.image_name - tag = try(coalesce(var.mongodb.image_tag), local.default_tags[var.mongodb.image_name]) - node_selector = var.mongodb.node_selector - image_pull_secrets = var.mongodb.image_pull_secrets - replicas_number = var.mongodb.replicas_number + image = var.mongodb.image_name + tag = try(coalesce(var.mongodb.image_tag), local.default_tags[var.mongodb.image_name]) + node_selector = var.mongodb.node_selector + image_pull_secrets = var.mongodb.image_pull_secrets + replicas = var.mongodb.replicas + helm_chart_repository = try(coalesce(var.mongodb.helm_chart_repository), var.helm_charts.mongodb.repository) + helm_chart_version = try(coalesce(var.mongodb.helm_chart_version), var.helm_charts.mongodb.version) } persistent_volume = null } @@ -181,10 +183,10 @@ locals { host = module.mongodb.host port = module.mongodb.port credentials = module.mongodb.user_credentials - certificates = module.mongodb.user_certificate endpoints = module.mongodb.endpoints - number_of_replicas = var.mongodb.replicas_number + number_of_replicas = var.mongodb.replicas allow_insecure_tls = true + #certificates = module.mongodb.user_certificate } shared = var.shared_storage != null ? var.shared_storage : { host_path = abspath("data") diff --git a/infrastructure/quick-deploy/localhost/variables.tf b/infrastructure/quick-deploy/localhost/variables.tf index 79458edf5..089e042d6 100644 --- a/infrastructure/quick-deploy/localhost/variables.tf +++ b/infrastructure/quick-deploy/localhost/variables.tf @@ -135,11 +135,13 @@ variable "rabbitmq" { variable "mongodb" { description = "Parameters of MongoDB" type = object({ - image_name = optional(string, "mongo") - image_tag = optional(string) - node_selector = optional(any, {}) - image_pull_secrets = optional(string, "") - replicas_number = optional(number, 1) + image_name = optional(string, "bitnami/mongodb") + image_tag = optional(string) + node_selector = optional(any, {}) + image_pull_secrets = optional(string, "") + replicas = optional(number, 1) + helm_chart_repository = optional(string) + helm_chart_version = optional(string) }) default = {} } diff --git a/tools/access-mongo-as-admin.sh b/tools/access-mongo-as-admin.sh deleted file mode 100755 index d5bfe7dfb..000000000 --- a/tools/access-mongo-as-admin.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/sh - -# ACESS to monogodb as admin -MPASS="$(kubectl get secret -n armonik mongodb-admin -o jsonpath="{.data.password}" | base64 --decode)" -MUSER="$(kubectl get secret -n armonik mongodb-admin -o jsonpath="{.data.username}" | base64 --decode)" -kubectl get secret -n armonik mongodb-user-certificates -o jsonpath="{.data.chain\.pem}" | base64 --decode > ./mongodb_chain.pem -MONGO_IP="$(kubectl get svc mongodb-0 -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true)" -docker run -it -v "$(pwd)/mongodb_chain.pem:/chain.pem" --rm rtsp/mongosh mongosh --tlsCAFile /chain.pem --tlsAllowInvalidCertificates --tlsAllowInvalidHostnames --tls -u "$MUSER" -p "$MPASS" "mongodb://$MONGO_IP:27017" diff --git a/tools/access-mongo-as-user.sh b/tools/access-mongo-as-user.sh deleted file mode 100755 index 354a63d24..000000000 --- a/tools/access-mongo-as-user.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/sh - -# ACESS to monogodb as user -MPASS="$(kubectl get secret -n armonik mongodb-user -o jsonpath="{.data.password}" | base64 --decode)" -MUSER="$(kubectl get secret -n armonik mongodb-user -o jsonpath="{.data.username}" | base64 --decode)" -kubectl get secret -n armonik mongodb-user-certificates -o jsonpath="{.data.chain\.pem}" | base64 --decode > ./mongodb_chain.pem -MONGO_IP="$(kubectl get svc mongodb-0 -n armonik -o custom-columns="IP:.spec.clusterIP" --no-headers=true)" -docker run -it -v "$(pwd)/mongodb_chain.pem:/chain.pem" --rm rtsp/mongosh mongosh --tlsCAFile /chain.pem --tlsAllowInvalidCertificates --tlsAllowInvalidHostnames --tls -u "$MUSER" -p "$MPASS" "mongodb://$MONGO_IP:27017/database" #--eval 'db.serverStatus()' diff --git a/tools/access-mongo-from-kubernetes-as-admin.sh b/tools/access-mongo-from-kubernetes-as-admin.sh index 94b360aab..efea18b5e 100755 --- a/tools/access-mongo-from-kubernetes-as-admin.sh +++ b/tools/access-mongo-from-kubernetes-as-admin.sh @@ -44,7 +44,7 @@ kubectl run -it --rm -n armonik mongoshclient --image=rtsp/mongosh --overrides=' "-c" ], "args": [ - "mongosh --tlsCAFile /mongodb/chain.pem --tlsAllowInvalidCertificates --tlsAllowInvalidHostnames --tls -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD mongodb://mongodb:27017" + "mongosh --tlsCAFile /mongodb/chain.pem --tlsAllowInvalidCertificates --tlsAllowInvalidHostnames --tls -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD mongodb+srv://mongodb-armonik-headless.armonik.svc.cluster.local/" ], "env": [ { diff --git a/tools/access-mongo-from-kubernetes-as-user.sh b/tools/access-mongo-from-kubernetes-as-user.sh new file mode 100755 index 000000000..75127b37b --- /dev/null +++ b/tools/access-mongo-from-kubernetes-as-user.sh @@ -0,0 +1,88 @@ +#! /bin/sh + +set -e + +cat < useful for AWS installation ***** +********************************************************************************************************************** + +- You can execute requests ex : +- Display all TaskData : + db.TaskData.find().limit(3).pretty() +- Filter by session / output : + db.TaskData.find({ SessionId: { \$eq : '7eafe4e3-0aa2-46ef-8ce6-bf9e365c5449' }, ExpectedOutputIds: { \$eq : 'a600dca5-b672-4177-9b4a-880dbcefee4e'}}).pretty() + +more informations here : https://www.mongodb.com/docs/manual/reference/method/db.collection.find/ + +EOF + +kubectl run -it --rm -n armonik mongoshclient --image=rtsp/mongosh --overrides=' +{ + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "creationTimestamp": null, + "labels": { + "run": "mongoshclient" + }, + "name": "mongoshclient", + "namespace": "armonik" + }, + "spec": { + "containers": [ + { + "name": "mongosh", + "image": "rtsp/mongosh", + "stdin": true, + "tty": true, + "command": [ + "bash", + "-c" + ], + "args": [ + "mongosh --tlsCAFile /mongodb/chain.pem --tlsAllowInvalidCertificates --tlsAllowInvalidHostnames --tls -u $MONGO_USERNAME -p $MONGO_USER_PASSWORD mongodb+srv://mongodb-armonik-headless.armonik.svc.cluster.local/database" + ], + "env": [ + { + "name": "MONGO_USERNAME", + "valueFrom": { + "secretKeyRef": { + "name": "mongodb-user", + "key": "username" + } + } + }, + { + "name": "MONGO_USER_PASSWORD", + "valueFrom": { + "secretKeyRef": { + "name": "mongodb-user", + "key": "password" + } + } + } + ], + "volumeMounts": [ + { + "name": "mongodb-secret-volume", + "mountPath": "/mongodb/" + } + ], + "resources": {} + } + ], + "volumes": [ + { + "name": "mongodb-secret-volume", + "secret": { + "secretName": "mongodb" + } + } + ], + "dnsPolicy": "ClusterFirst", + "restartPolicy": "Always" + }, + "status": {} +} +' diff --git a/versions.tfvars.json b/versions.tfvars.json index 2896048a6..285785f26 100644 --- a/versions.tfvars.json +++ b/versions.tfvars.json @@ -1,7 +1,7 @@ { "armonik_versions": { "armonik": "2.20.0", - "infra": "0.4.3", + "infra": "0.5.0-pre-2-d48d300", "infra_plugins": "0.1.0", "core": "0.24.2", "api": "3.18.1", @@ -58,6 +58,7 @@ "public.ecr.aws/eks-distro/kubernetes-csi/external-provisioner": "v4.0.0-eks-1-29-8", "symptoma/activemq": "5.18.3", "mongo": "7.0.5", + "bitnami/mongodb": "7.0.12-debian-12-r0", "redis": "7.2.5-alpine", "minio/minio": "RELEASE.2024-05-10T01-41-38Z", "datalust/seq": "2024.3", @@ -68,7 +69,7 @@ "rtsp/mongosh": "2.2.6", "nginxinc/nginx-unprivileged": "1.25.5-alpine-slim", "datalust/seqcli": "2024.3", - "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner": "v4.0.2", + "registry.k8s.io/sig-storage/nfs-subdir-external-provisioner": "v4.0.2", "bitnami/rabbitmq": "3.12.14", "ghcr.io/chaos-mesh/chaos-mesh": "v2.6.3", "ghcr.io/chaos-mesh/chaos-daemon": "v2.6.3", @@ -81,6 +82,7 @@ "termination_handler" : {"repository" : "https://aws.github.io/eks-charts" , "version" : "0.21.0" }, "efs_csi_driver" : { "repository" :"https://kubernetes-sigs.github.io/aws-efs-csi-driver/" , "version": "2.5.7" }, "rabbitmq" : { "repository" : "https://charts.bitnami.com/bitnami" , "version" : "13.0.2"}, - "chaos_mesh" : { "repository" : "https://charts.chaos-mesh.org" , "version" : "2.6.3"} + "chaos_mesh" : { "repository" : "https://charts.chaos-mesh.org" , "version" : "2.6.3"}, + "mongodb" : { "repository": "oci://registry-1.docker.io/bitnamicharts", "version" : "15.6.12"} } }