diff --git a/.gitignore b/.gitignore index 647518c..4aaa98a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ dev.db* rpc.toml daemon.toml grpcurl* +test/.terraform* +test/local.tfstate* diff --git a/bootstap/daemon/config.tf b/bootstap/daemon/config.tf new file mode 100644 index 0000000..af8d1f0 --- /dev/null +++ b/bootstap/daemon/config.tf @@ -0,0 +1,17 @@ +resource "kubernetes_config_map_v1" "fabric_daemon_config" { + metadata { + name = local.configmap_name + namespace = var.namespace + } + + data = { + "daemon.toml" = "${templatefile( + "${path.module}/daemon.toml", + { + broker_urls = var.broker_urls + } + )}" + } +} + + diff --git a/bootstap/daemon/deployment.tf b/bootstap/daemon/deployment.tf new file mode 100644 index 0000000..23a5047 --- /dev/null +++ b/bootstap/daemon/deployment.tf @@ -0,0 +1,83 @@ +locals { + role = "fabric-daemon" +} + +resource "kubernetes_deployment_v1" "daemon" { + wait_for_rollout = false + + metadata { + labels = { + role = local.role + } + name = "fabric-daemon" + namespace = var.namespace + } + + spec { + replicas = 1 + + selector { + match_labels = { + role = local.role + } + } + + template { + metadata { + labels = { + role = local.role + } + } + + spec { + container { + name = "daemon" + image = var.image + + env { + name = "DAEMON_CONFIG" + value = "/fabric/daemon.toml" + } + + port { + container_port = local.port + } + + volume_mount { + name = "config" + mount_path = "/fabric" + } + + resources { + limits = { + cpu = var.resources.limits.cpu + memory = var.resources.limits.memory + } + requests = { + cpu = var.resources.requests.cpu + memory = var.resources.requests.memory + } + } + } + + volume { + name = "config" + config_map { + name = local.configmap_name + } + } + + dynamic "toleration" { + for_each = var.tolerations + + content { + effect = toleration.value.effect + key = toleration.value.key + operator = toleration.value.operator + value = toleration.value.value + } + } + } + } + } +} diff --git a/bootstap/daemon/main.tf b/bootstap/daemon/main.tf new file mode 100644 index 0000000..8a068fd --- /dev/null +++ b/bootstap/daemon/main.tf @@ -0,0 +1,70 @@ +locals { + configmap_name = "fabric-daemon-config" + port = 5000 +} + +variable "namespace" { + type = string +} + +variable "image" { + type = string +} + +variable "broker_urls" { + type = string + description = "Comma separated values of the queue broker urls." +} + +variable "tolerations" { + type = list(object({ + effect = string + key = string + operator = string + value = string + })) + default = [ + { + effect = "NoSchedule" + key = "demeter.run/compute-profile" + operator = "Equal" + value = "general-purpose" + }, + { + effect = "NoSchedule" + key = "demeter.run/compute-arch" + operator = "Equal" + value = "x86" + }, + { + effect = "NoSchedule" + key = "demeter.run/availability-sla" + operator = "Equal" + value = "consistent" + } + + ] +} + +variable "resources" { + type = object({ + limits = object({ + cpu = optional(string) + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + }) + default = { + requests = { + cpu = "100m" + memory = "500Mi" + } + limits = { + cpu = "500m" + memory = "500Mi" + } + } +} diff --git a/bootstap/daemon/rbac.tf b/bootstap/daemon/rbac.tf new file mode 100644 index 0000000..eec97a3 --- /dev/null +++ b/bootstap/daemon/rbac.tf @@ -0,0 +1,27 @@ +resource "kubernetes_cluster_role" "cluster_role" { + metadata { + name = var.namespace + } + + rule { + api_groups = ["*"] + resources = ["*"] + verbs = ["*"] + } +} + +resource "kubernetes_cluster_role_binding" "cluster_role_binding" { + metadata { + name = var.namespace + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = var.namespace + } + subject { + kind = "ServiceAccount" + name = "default" + namespace = var.namespace + } +} diff --git a/bootstap/queue/console.tf b/bootstap/queue/console.tf new file mode 100644 index 0000000..7a06af4 --- /dev/null +++ b/bootstap/queue/console.tf @@ -0,0 +1,98 @@ +locals { + console_instance_name = "fabric-queue-console" + console_configmap_name = "${local.console_instance_name}-config" +} + +resource "kubernetes_config_map_v1" "fabric_queue_console_config" { + metadata { + name = local.console_configmap_name + namespace = var.namespace + } + + data = { + "config.yml" = "${templatefile( + "${path.module}/console_config.yml", + { + instance_name = var.instance_name, + kafka_external_port = local.kafka_external_port, + schema_registry_external_port = local.schema_registry_external_port, + admin_api_external_port = local.admin_api_external_port + } + )}" + } +} + +resource "kubernetes_deployment_v1" "fabric_queue_console" { + wait_for_rollout = false + depends_on = [kubernetes_config_map_v1.fabric_queue_console_config] + + metadata { + name = local.console_instance_name + namespace = var.namespace + } + + spec { + replicas = 1 + + selector { + match_labels = { + "demeter.run/instance" : local.console_instance_name + } + } + + template { + metadata { + labels = { + "demeter.run/instance" : local.console_instance_name + } + } + + spec { + container { + name = "main" + image = var.console_image + + env { + name = "CONFIG_FILEPATH" + value = "/etc/config/config.yml" + } + + volume_mount { + name = "config" + mount_path = "/etc/config" + } + + resources { + limits = { + cpu = var.console_resources.limits.cpu + memory = var.console_resources.limits.memory + } + requests = { + cpu = var.console_resources.requests.cpu + memory = var.console_resources.requests.memory + } + } + + } + + volume { + name = "config" + config_map { + name = local.console_configmap_name + } + } + + dynamic "toleration" { + for_each = var.tolerations + + content { + effect = toleration.value.effect + key = toleration.value.key + operator = toleration.value.operator + value = toleration.value.value + } + } + } + } + } +} diff --git a/bootstap/queue/console_config.yml b/bootstap/queue/console_config.yml new file mode 100644 index 0000000..64dcec1 --- /dev/null +++ b/bootstap/queue/console_config.yml @@ -0,0 +1,9 @@ +kafka: + brokers: ["${instance_name}:${kafka_external_port}"] + schemaRegistry: + enabled: true + urls: ["http://${instance_name}:${schema_registry_external_port}"] +redpanda: + adminApi: + enabled: true + urls: ["http://${instance_name}:${admin_api_external_port}"] diff --git a/bootstap/queue/main.tf b/bootstap/queue/main.tf new file mode 100644 index 0000000..5bf898d --- /dev/null +++ b/bootstap/queue/main.tf @@ -0,0 +1,108 @@ +locals { + kafka_internal_port = 9092 + kafka_external_port = 19092 + pandaproxy_internal_port = 8082 + pandaproxy_external_port = 18082 + schema_registry_internal_port = 8081 + schema_registry_external_port = 18081 + rpc_port = 33145 + admin_api_external_port = 19644 +} + +variable "namespace" { + type = string +} + +variable "instance_name" { + type = string +} + +variable "topology_zone" { + type = string + default = null +} + +variable "image" { + type = string +} + +variable "console_image" { + type = string + default = "docker.redpanda.com/redpandadata/console:v2.3.1" +} + +variable "resources" { + type = object({ + limits = object({ + cpu = optional(string) + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + }) + default = { + requests = { + cpu = "100m" + memory = "1G" + } + limits = { + cpu = "4" + memory = "1G" + } + } +} + +variable "console_resources" { + type = object({ + limits = object({ + cpu = optional(string) + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + }) + default = { + requests = { + cpu = "100m" + memory = "500Mi" + } + limits = { + cpu = "500m" + memory = "500Mi" + } + } +} + +variable "tolerations" { + type = list(object({ + effect = string + key = string + operator = string + value = string + })) + default = [ + { + effect = "NoSchedule" + key = "demeter.run/compute-profile" + operator = "Equal" + value = "general-purpose" + }, + { + effect = "NoSchedule" + key = "demeter.run/compute-arch" + operator = "Equal" + value = "x86" + }, + { + effect = "NoSchedule" + key = "demeter.run/availability-sla" + operator = "Equal" + value = "consistent" + } + + ] +} diff --git a/bootstap/queue/service.tf b/bootstap/queue/service.tf new file mode 100644 index 0000000..558c977 --- /dev/null +++ b/bootstap/queue/service.tf @@ -0,0 +1,70 @@ +resource "kubernetes_service_v1" "fabric_queue_service" { + metadata { + namespace = var.namespace + name = var.instance_name + } + + spec { + type = "ClusterIP" + + port { + name = "schema-registry-internal" + protocol = "TCP" + port = local.schema_registry_internal_port + target_port = local.schema_registry_internal_port + } + + port { + name = "schema-registry-external" + protocol = "TCP" + port = local.schema_registry_external_port + target_port = local.schema_registry_external_port + } + + port { + name = "pandaproxy-internal" + protocol = "TCP" + port = local.pandaproxy_internal_port + target_port = local.pandaproxy_internal_port + } + + port { + name = "pandaproxy-external" + protocol = "TCP" + port = local.pandaproxy_external_port + target_port = local.pandaproxy_external_port + } + + port { + name = "kafka-internal" + protocol = "TCP" + port = local.kafka_internal_port + target_port = local.kafka_internal_port + } + + port { + name = "kafka-external" + protocol = "TCP" + port = local.kafka_external_port + target_port = local.kafka_external_port + } + + port { + name = "rpc" + protocol = "TCP" + port = local.rpc_port + target_port = local.rpc_port + } + + port { + name = "admin-api-external" + port = local.admin_api_external_port + protocol = "TCP" + target_port = 9644 + } + + selector = { + "demeter.run/instance" = var.instance_name + } + } +} diff --git a/bootstap/queue/sts.tf b/bootstap/queue/sts.tf new file mode 100644 index 0000000..c336749 --- /dev/null +++ b/bootstap/queue/sts.tf @@ -0,0 +1,184 @@ +locals { + args = [ + "redpanda", + "start", + "--kafka-addr", + "internal://${var.instance_name}-0:${local.kafka_internal_port},external://0.0.0.0:${local.kafka_external_port}", + "--advertise-kafka-addr", + "internal://${var.instance_name}:${local.kafka_internal_port},external://${var.instance_name}:${local.kafka_external_port}", + "--pandaproxy-addr", + "internal://${var.instance_name}-0:${local.pandaproxy_internal_port},external://0.0.0.0:${local.pandaproxy_external_port}", + "--advertise-pandaproxy-addr", + "internal://${var.instance_name}:${local.pandaproxy_internal_port},external://${var.instance_name}:${local.pandaproxy_external_port}", + "--schema-registry-addr", + "internal://${var.instance_name}-0:${local.schema_registry_internal_port},external://0.0.0.0:${local.schema_registry_external_port}", + "--rpc-addr", + "${var.instance_name}-0:${local.rpc_port}", + "--advertise-rpc-addr", "${var.instance_name}:${local.rpc_port}", + "--smp", "1", + "--reserve-memory", "0M", + ] + + create_topic_job_name = "${var.instance_name}-create-topic" +} + +resource "kubernetes_stateful_set_v1" "queue_main" { + metadata { + name = var.instance_name + namespace = var.namespace + } + spec { + replicas = 1 + service_name = "fabric-queue" + selector { + match_labels = { + "demeter.run/instance" = var.instance_name + "role" = "fabric-queue-leader" + } + } + + template { + metadata { + labels = { + "demeter.run/instance" = var.instance_name + "role" = "fabric-queue-leader" + } + } + spec { + dynamic "affinity" { + for_each = var.topology_zone != null ? toset([1]) : toset([]) + + content { + node_affinity { + required_during_scheduling_ignored_during_execution { + node_selector_term { + match_expressions { + key = "topology.kubernetes.io/zone" + operator = "In" + values = [var.topology_zone] + } + } + } + } + } + } + + security_context { + fs_group = 1000 + } + + container { + name = "main" + image = var.image + args = concat(local.args, ["--node-id", "0"]) + image_pull_policy = "Always" + + resources { + limits = { + cpu = var.resources.limits.cpu + memory = var.resources.limits.memory + } + requests = { + cpu = var.resources.requests.cpu + memory = var.resources.requests.memory + } + } + + # lifecycle { + # post_start { + # exec { + # # command = [ + # # "rpk", + # # "-X", + # # "brokers=${var.instance_name}:${local.kafka_external_port}", + # # "topic", + # # "create", + # # "events" + # # ] + # + # command = [ + # "sh", "-c", <<-EOT + # sleep 1 + # rpk -X brokers=fabric-queue-0:9092 topic create events + # EOT + # ] + # } + # } + # } + } + + dynamic "toleration" { + for_each = var.tolerations + + content { + effect = toleration.value.effect + key = toleration.value.key + operator = toleration.value.operator + value = toleration.value.value + } + } + } + } + } +} + + +resource "kubernetes_job_v1" "fabric_queue_create_topic" { + depends_on = [kubernetes_stateful_set_v1.queue_main] + + metadata { + name = local.create_topic_job_name + namespace = var.namespace + } + spec { + template { + metadata { + labels = { + "demeter.run/instance" = local.create_topic_job_name + } + } + spec { + security_context { + fs_group = 1000 + } + + container { + name = "main" + image = var.image + command = [ + "rpk", + "-X", + "brokers=${var.instance_name}:${local.kafka_external_port}", + "topic", + "create", + "events" + ] + image_pull_policy = "Always" + + resources { + limits = { + cpu = var.resources.limits.cpu + memory = var.resources.limits.memory + } + requests = { + cpu = var.resources.requests.cpu + memory = var.resources.requests.memory + } + } + } + + dynamic "toleration" { + for_each = var.tolerations + + content { + effect = toleration.value.effect + key = toleration.value.key + operator = toleration.value.operator + value = toleration.value.value + } + } + } + } + } + +} diff --git a/bootstap/rpc/config.tf b/bootstap/rpc/config.tf new file mode 100644 index 0000000..84e5102 --- /dev/null +++ b/bootstap/rpc/config.tf @@ -0,0 +1,19 @@ +resource "kubernetes_config_map_v1" "fabric_rpc_config" { + metadata { + name = local.configmap_name + namespace = var.namespace + } + + data = { + "rpc.toml" = "${templatefile( + "${path.module}/rpc.toml", + { + port = local.port, + db_path = "cache.db", + broker_urls = var.broker_urls + } + )}" + } +} + + diff --git a/bootstap/rpc/deployment.tf b/bootstap/rpc/deployment.tf new file mode 100644 index 0000000..917dea2 --- /dev/null +++ b/bootstap/rpc/deployment.tf @@ -0,0 +1,83 @@ +locals { + role = "fabric-rpc" +} + +resource "kubernetes_deployment_v1" "rpc" { + wait_for_rollout = false + + metadata { + labels = { + role = local.role + } + name = "fabric-rpc" + namespace = var.namespace + } + + spec { + replicas = 1 + + selector { + match_labels = { + role = local.role + } + } + + template { + metadata { + labels = { + role = local.role + } + } + + spec { + container { + name = "rpc" + image = var.image + + env { + name = "RPC_CONFIG" + value = "/fabric/rpc.toml" + } + + port { + container_port = local.port + } + + volume_mount { + name = "config" + mount_path = "/fabric" + } + + resources { + limits = { + cpu = var.resources.limits.cpu + memory = var.resources.limits.memory + } + requests = { + cpu = var.resources.requests.cpu + memory = var.resources.requests.memory + } + } + } + + volume { + name = "config" + config_map { + name = local.configmap_name + } + } + + dynamic "toleration" { + for_each = var.tolerations + + content { + effect = toleration.value.effect + key = toleration.value.key + operator = toleration.value.operator + value = toleration.value.value + } + } + } + } + } +} diff --git a/bootstap/rpc/main.tf b/bootstap/rpc/main.tf new file mode 100644 index 0000000..ae86006 --- /dev/null +++ b/bootstap/rpc/main.tf @@ -0,0 +1,70 @@ +locals { + configmap_name = "fabric-rpc-config" + port = 5050 +} + +variable "namespace" { + type = string +} + +variable "image" { + type = string +} + +variable "broker_urls" { + type = string + description = "Comma separated values of the queue broker urls." +} + +variable "tolerations" { + type = list(object({ + effect = string + key = string + operator = string + value = string + })) + default = [ + { + effect = "NoSchedule" + key = "demeter.run/compute-profile" + operator = "Equal" + value = "general-purpose" + }, + { + effect = "NoSchedule" + key = "demeter.run/compute-arch" + operator = "Equal" + value = "x86" + }, + { + effect = "NoSchedule" + key = "demeter.run/availability-sla" + operator = "Equal" + value = "consistent" + } + + ] +} + +variable "resources" { + type = object({ + limits = object({ + cpu = optional(string) + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + }) + default = { + requests = { + cpu = "100m" + memory = "500Mi" + } + limits = { + cpu = "500m" + memory = "500Mi" + } + } +} diff --git a/bootstap/rpc/service.tf b/bootstap/rpc/service.tf new file mode 100644 index 0000000..4d82f8d --- /dev/null +++ b/bootstap/rpc/service.tf @@ -0,0 +1,21 @@ +resource "kubernetes_service_v1" "fabric_rpc_service" { + metadata { + namespace = var.namespace + name = "fabric-rpc" + } + + spec { + type = "ClusterIP" + + port { + name = "grpc" + port = local.port + protocol = "TCP" + target_port = local.port + } + + selector = { + role = local.role + } + } +} diff --git a/bootstap/services/main.tf b/bootstap/services/main.tf new file mode 100644 index 0000000..e0a17be --- /dev/null +++ b/bootstap/services/main.tf @@ -0,0 +1,76 @@ +variable "namespace" { + type = string +} + +variable "ingress_class_name" { + type = string + default = "nginx" +} + +variable "dns_zone" { + type = string + default = "demeter.run" +} + +resource "kubernetes_service_v1" "fabric-queue-load-balancer" { + metadata { + namespace = var.namespace + name = "fabric-queue-load-balancer" + annotations = { + "beta.kubernetes.io/aws-load-balancer-nlb-target-type" = "instance" + "service.beta.kubernetes.io/aws-load-balancer-scheme" = "internet-facing" + "service.beta.kubernetes.io/aws-load-balancer-type" = "external" + } + } + + spec { + type = "LoadBalancer" + load_balancer_class = "service.k8s.aws/nlb" + + port { + protocol = "TCP" + port = 9092 + target_port = 19092 + } + + selector = { + "role" = "fabric-queue-leader" + } + } +} + +resource "kubernetes_ingress_v1" "fabric-rpc-ingress" { + wait_for_load_balancer = true + metadata { + name = "fabric-rpc-ingress" + namespace = var.namespace + annotations = { + "cert-manager.io/cluster-issuer" = "letsencrypt" + } + } + + spec { + ingress_class_name = var.ingress_class_name + + rule { + host = "rpc.${var.dns_zone}" + http { + path { + path = "/" + + backend { + service { + name = "fabric-rpc" + port { + number = 5050 + } + } + } + } + } + } + tls { + hosts = ["rpc.${var.dns_zone}"] + } + } +}