From e3cf6dde4c932083318db5e391a0cce5ef7a237e Mon Sep 17 00:00:00 2001 From: "Klaus @ LambdaClass" <18153834+klaus993@users.noreply.github.com> Date: Mon, 19 Aug 2024 18:18:28 +0000 Subject: [PATCH] Add databases to terraform module (#18) * Parametrize cluster zones * Add DBs with private IPs in the GKE cluster network * Add db configuration in module example * Parametrize prevent_destroy for kms key --- infra/terraform/dev/us-east4/providers.tf | 3 + infra/terraform/dev/us-east4/zk_stack.tf | 29 +++++- infra/terraform/modules/zk_stack/db.tf | 97 +++++++++++++++++++ infra/terraform/modules/zk_stack/gke.tf | 2 +- infra/terraform/modules/zk_stack/kms.tf | 2 +- infra/terraform/modules/zk_stack/variables.tf | 55 +++++++++++ 6 files changed, 181 insertions(+), 7 deletions(-) create mode 100644 infra/terraform/modules/zk_stack/db.tf diff --git a/infra/terraform/dev/us-east4/providers.tf b/infra/terraform/dev/us-east4/providers.tf index a993a52..f79eac7 100644 --- a/infra/terraform/dev/us-east4/providers.tf +++ b/infra/terraform/dev/us-east4/providers.tf @@ -10,3 +10,6 @@ terraform { provider "google" { project = "zksync-413615" } + +provider "aws" { +} diff --git a/infra/terraform/dev/us-east4/zk_stack.tf b/infra/terraform/dev/us-east4/zk_stack.tf index 002d69e..c5b3c5a 100644 --- a/infra/terraform/dev/us-east4/zk_stack.tf +++ b/infra/terraform/dev/us-east4/zk_stack.tf @@ -1,13 +1,15 @@ module "zk_stack" { source = "git::https://github.com/lambdaclass/zk_stack//infra/terraform/modules/zk_stack" - region = "us-east4" - cluster_name = "zksync-dev-02" - project_id = "zksync-413615" + region = "us-east4" + cluster_name = "zksync-dev-99" + project_id = "zksync-413615" + cluster_locations = ["us-east4-a", "us-east4-b"] # Nodes configs - cpu_nodes_locations = "us-central1-a" - gpu_nodes_locations = "us-central1-c" + cpu_machine_type = "c3-standard-4" + cpu_nodes_locations = "us-east4-a" + gpu_nodes_locations = "us-east4-b" # DNS configuration aws_dns_zone = "zk-stack.lambdaclass.com" @@ -34,4 +36,21 @@ module "zk_stack" { prover_object_store_bucket_name = "prover-object-store-dev-2" snapshots_object_store_bucket_name = "snapshots-object-store-dev-2" prover_setup_data_bucket_name = "prover-setup-data-2" + + # Central DB configuration + db_size = "db-custom-1-3840" + db_disk_size_gb = "20" + sql_user = "admin" + sql_password = var.sql_password + # Prover DB configuration + prover_db_size = "db-custom-1-3840" + prover_db_disk_size_gb = "20" + prover_sql_user = "admin" + prover_sql_password = var.prover_sql_password +} + +variable "sql_password" { +} + +variable "prover_sql_password" { } diff --git a/infra/terraform/modules/zk_stack/db.tf b/infra/terraform/modules/zk_stack/db.tf new file mode 100644 index 0000000..6f03609 --- /dev/null +++ b/infra/terraform/modules/zk_stack/db.tf @@ -0,0 +1,97 @@ +# VPC Peering between Cloud SQL and VPC +resource "google_compute_global_address" "peering_default_ip_range" { + name = "peering-default-ip-range" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 20 + network = google_compute_network.gke-cluster-network.id + + depends_on = [ + google_compute_subnetwork.gke-cluster-subnetwork + ] +} + +resource "google_service_networking_connection" "databases" { + network = google_compute_network.gke-cluster-network.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [ + google_compute_global_address.peering_default_ip_range.name + ] +} + +resource "google_compute_network_peering_routes_config" "dbs_peering_routes" { + peering = google_service_networking_connection.databases.peering + network = google_compute_network.gke-cluster-network.name + import_custom_routes = true + export_custom_routes = true +} + +# Central DB +resource "google_sql_database_instance" "general" { + name = var.cluster_name + database_version = "POSTGRES_14" + region = var.region + + deletion_protection = false + + settings { + tier = var.db_size # "db-custom-4-15360" + activation_policy = "ALWAYS" + availability_type = "ZONAL" + disk_type = "PD_SSD" + disk_size = var.db_disk_size_gb + disk_autoresize = true + ip_configuration { + ipv4_enabled = false + private_network = google_compute_network.gke-cluster-network.id + } + backup_configuration { + enabled = false + } + } + + depends_on = [ + google_service_networking_connection.databases + ] +} + +resource "google_sql_user" "general" { + name = var.sql_user + instance = google_sql_database_instance.general.name + password = var.sql_password +} + +# Prover DB +resource "google_sql_database_instance" "prover" { + name = "${var.cluster_name}-prover" + database_version = "POSTGRES_14" + region = var.region + + deletion_protection = false + + settings { + tier = var.prover_db_size + activation_policy = "ALWAYS" + availability_type = "ZONAL" + disk_type = "PD_SSD" + disk_size = var.prover_db_disk_size_gb + disk_autoresize = true + ip_configuration { + ipv4_enabled = false + private_network = google_compute_network.gke-cluster-network.id + } + backup_configuration { + enabled = false + } + } + + depends_on = [ + google_service_networking_connection.databases + ] +} + +resource "google_sql_user" "prover" { + name = var.prover_sql_user + instance = google_sql_database_instance.prover.name + password = var.prover_sql_password +} diff --git a/infra/terraform/modules/zk_stack/gke.tf b/infra/terraform/modules/zk_stack/gke.tf index e3b3e92..0ff04fe 100644 --- a/infra/terraform/modules/zk_stack/gke.tf +++ b/infra/terraform/modules/zk_stack/gke.tf @@ -4,7 +4,7 @@ module "zk-stack-gke-cluster" { project_id = var.project_id name = var.cluster_name region = var.region - zones = ["${var.region}-a", "${var.region}-b", "${var.region}-c"] + zones = var.cluster_locations release_channel = "STABLE" # Network config diff --git a/infra/terraform/modules/zk_stack/kms.tf b/infra/terraform/modules/zk_stack/kms.tf index a80f2a3..df1ad11 100644 --- a/infra/terraform/modules/zk_stack/kms.tf +++ b/infra/terraform/modules/zk_stack/kms.tf @@ -9,6 +9,6 @@ resource "google_kms_crypto_key" "k8s-secrets-encryption-key" { rotation_period = "7776000s" lifecycle { - prevent_destroy = true + prevent_destroy = var.encryption_key_prevent_destroy } } diff --git a/infra/terraform/modules/zk_stack/variables.tf b/infra/terraform/modules/zk_stack/variables.tf index 74c7376..07ca069 100644 --- a/infra/terraform/modules/zk_stack/variables.tf +++ b/infra/terraform/modules/zk_stack/variables.tf @@ -13,6 +13,11 @@ variable "project_id" { description = "ID of the GCP Project to deploy all the infrastructure" } +variable "cluster_locations" { + type = list + description = "List of GCP zones (inside the region you specified) where the cluster will be" +} + variable "cpu_machine_type" { type = string default = "c3-highmem-22" @@ -184,3 +189,53 @@ variable "prover_setup_data_bucket_name" { default = "prover-setup-data" description = "GCS Bucket name for the prover setup data bucket" } + +variable "db_size" { + type = string + default = "db-custom-4-15360" + description = "Cloud SQL General DB size/type" +} + +variable "db_disk_size_gb" { + type = string + default = "100" + description = "Cloud SQL General DB disk size in GiB" +} + +variable "sql_user" { + type = string + description = "Cloud SQL General DB username" +} + +variable "sql_password" { + type = string + description = "Cloud SQL General DB password" +} + +variable "prover_db_size" { + type = string + default = "db-custom-4-15360" + description = "Cloud SQL Prover DB size/type" +} + +variable "prover_db_disk_size_gb" { + type = string + default = "100" + description = "Cloud SQL Prover DB disk size in GiB" +} + +variable "prover_sql_user" { + type = string + description = "Cloud SQL Prover DB username" +} + +variable "prover_sql_password" { + type = string + description = "Cloud SQL Prover DB password" +} + +variable "encryption_key_prevent_destroy" { + type = bool + default = true + description = "Whether to prevent destroying the GCP KMS decrpytion key for Kubernetes data" +}