From c190aacb304b9de30ef7058b597e823353fa4f24 Mon Sep 17 00:00:00 2001 From: DaMandal0rian <3614052+DaMandal0rian@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:45:31 +0300 Subject: [PATCH] add auto domain to infra PT.1 (#311) * add auto domain to infra add auto add DNS records minor fixes domain label and ID add operator ID decouple 2 separate domains for evm and autoid add more changes to domains Fix Fix traefik labels and remove auto extra chains in configs fix merge conflict * bump traefik release * fix more references * remove relayer ids --- resources/devnet/main.tf | 37 +++- resources/devnet/outputs.tf | 10 + resources/devnet/variables.tf | 8 +- resources/gemini-3h/main.tf | 35 ++++ resources/gemini-3h/outputs.tf | 10 + resources/gemini-3h/variables.tf | 6 +- .../create_autoid_node_compose_file.sh | 176 ++++++++++++++++ ...eate_bootstrap_node_autoid_compose_file.sh | 197 ++++++++++++++++++ .../create_bootstrap_node_evm_compose_file.sh | 5 +- .../create_domain_node_compose_file.sh | 5 +- .../hetzner/bootstrap_node_evm_provisioner.tf | 8 +- .../hetzner/domain_node_provisioner.tf | 12 +- templates/terraform/hetzner/variables.tf | 2 +- .../autoid_node_provisioner.tf | 184 ++++++++++++++++ .../bootstrap_node_autoid_provisioner.tf | 168 +++++++++++++++ templates/terraform/network-primitives/dns.tf | 42 +++- .../domain_node_provisioner.tf | 7 +- .../terraform/network-primitives/instances.tf | 126 +++++++++++ .../terraform/network-primitives/outputs.tf | 33 +++ .../terraform/network-primitives/variables.tf | 44 +++- 20 files changed, 1082 insertions(+), 33 deletions(-) create mode 100755 templates/scripts/create_autoid_node_compose_file.sh create mode 100755 templates/scripts/create_bootstrap_node_autoid_compose_file.sh create mode 100644 templates/terraform/network-primitives/autoid_node_provisioner.tf create mode 100644 templates/terraform/network-primitives/bootstrap_node_autoid_provisioner.tf diff --git a/resources/devnet/main.tf b/resources/devnet/main.tf index 17e1ced8..e0bbc92c 100644 --- a/resources/devnet/main.tf +++ b/resources/devnet/main.tf @@ -37,6 +37,23 @@ module "devnet" { disk-volume-type = var.disk_volume_type } + bootstrap-node-autoid-config = { + instance-type = var.instance_type["autoid_bootstrap"] + deployment-version = 1 + regions = var.aws_region + instance-count = var.instance_count["autoid_bootstrap"] + docker-org = "subspace" + docker-tag = "snapshot-2024-jan-23-2" + reserved-only = false + prune = false + genesis-hash = "" + dsn-listen-port = 30533 + node-dsn-port = 30433 + operator-port = 30334 + disk-volume-size = var.disk_volume_size + disk-volume-type = var.disk_volume_type + } + full-node-config = { instance-type = var.instance_type["full"] deployment-version = 0 @@ -73,7 +90,25 @@ module "devnet" { instance-count = var.instance_count["domain"] docker-org = "subspace" docker-tag = "snapshot-2024-jan-23-2" - domain-prefix = "domain" + domain-prefix = ["nova"] + reserved-only = false + prune = false + node-dsn-port = 30434 + enable-domains = true + domain-id = var.domain_id + domain-labels = var.domain_labels + disk-volume-size = var.disk_volume_size + disk-volume-type = var.disk_volume_type + } + + autoid-node-config = { + instance-type = var.instance_type["autoid"] + deployment-version = 0 + regions = var.aws_region + instance-count = var.instance_count["autoid"] + docker-org = "subspace" + docker-tag = "snapshot-2024-jan-23-2" + domain-prefix = ["autoid"] reserved-only = false prune = false node-dsn-port = 30434 diff --git a/resources/devnet/outputs.tf b/resources/devnet/outputs.tf index fbb6ea8f..4288d5e7 100644 --- a/resources/devnet/outputs.tf +++ b/resources/devnet/outputs.tf @@ -19,6 +19,11 @@ output "bootstrap-node-evm-ipv4-addresses" { description = "EVM Bootstrap node IPv4 Addresses" } +output "bootstrap-node-autoid-ipv4-addresses" { + value = module.devnet.bootstrap_node_autoid_public_ip + description = "AutoID Bootstrap node IPv4 Addresses" +} + output "rpc-node-ipv4-addresses" { value = module.devnet.rpc_node_public_ip description = "Domain node IPv4 Addresses" @@ -28,3 +33,8 @@ output "domain-node-ipv4-addresses" { value = module.devnet.domain_node_public_ip description = "Domain node IPv4 Addresses" } + +output "autoid-node-ipv4-addresses" { + value = module.devnet.autoid_node_public_ip + description = "AutoID node IPv4 Addresses" +} diff --git a/resources/devnet/variables.tf b/resources/devnet/variables.tf index afc53004..9114c851 100644 --- a/resources/devnet/variables.tf +++ b/resources/devnet/variables.tf @@ -6,13 +6,13 @@ variable "farmer_reward_address" { variable "domain_id" { description = "Domain ID" type = list(number) - default = [0] + default = [0, 1] } variable "domain_labels" { description = "Tag of the domain to run" type = list(string) - default = ["evm"] + default = ["nova", "autoid"] } variable "instance_type" { @@ -21,9 +21,11 @@ variable "instance_type" { bootstrap = "c6a.2xlarge" rpc = "m6a.xlarge" domain = "m6a.xlarge" + autoid = "m6a.xlarge" full = "m6a.xlarge" farmer = "c7i.2xlarge" evm_bootstrap = "m6a.xlarge" + autoid_bootstrap = "m6a.xlarge" } } @@ -48,9 +50,11 @@ variable "instance_count" { bootstrap = 2 rpc = 1 domain = 1 + autoid = 1 full = 0 farmer = 1 evm_bootstrap = 1 + autoid_bootstrap = 1 } } diff --git a/resources/gemini-3h/main.tf b/resources/gemini-3h/main.tf index 687854d2..b7788b77 100644 --- a/resources/gemini-3h/main.tf +++ b/resources/gemini-3h/main.tf @@ -36,6 +36,23 @@ module "gemini-3h" { disk-volume-type = var.disk_volume_type } + bootstrap-node-autoid-config = { + instance-type = var.instance_type["autoid_bootstrap"] + deployment-version = 0 + regions = var.aws_region + instance-count = var.instance_count["autoid_bootstrap"] + docker-org = "autonomys" + docker-tag = "gemini-3h-2024-jul-29" + reserved-only = false + prune = false + genesis-hash = "0c121c75f4ef450f40619e1fca9d1e8e7fbabc42c895bc4790801e85d5a91c34" + dsn-listen-port = 30533 + node-dsn-port = 30433 + operator-port = 30334 + disk-volume-size = var.disk_volume_size + disk-volume-type = var.disk_volume_type + } + rpc-squid-node-config = { instance-type = var.instance_type["rpc-squid"] deployment-version = 0 @@ -102,6 +119,24 @@ module "gemini-3h" { disk-volume-type = var.disk_volume_type } + autoid-node-config = { + instance-type = var.instance_type["autoid"] + deployment-version = 0 + regions = var.aws_region + instance-count = var.instance_count["autoid"] + docker-org = "subspace" + docker-tag = "gemini-3h-2024-may-06" + domain-prefix = ["autoid"] + reserved-only = false + prune = false + node-dsn-port = 30434 + enable-domains = true + domain-id = var.domain_id + domain-labels = var.domain_labels + disk-volume-size = var.disk_volume_size + disk-volume-type = var.disk_volume_type + } + farmer-node-config = { instance-type = var.instance_type["farmer"] deployment-version = 0 diff --git a/resources/gemini-3h/outputs.tf b/resources/gemini-3h/outputs.tf index 62af86ee..3238f766 100644 --- a/resources/gemini-3h/outputs.tf +++ b/resources/gemini-3h/outputs.tf @@ -24,11 +24,21 @@ output "bootstrap-node-evm-ipv4-addresses" { description = "EVM Bootstrap node IPv4 Addresses" } +output "bootstrap-node-autoid-ipv4-addresses" { + value = module.gemini-3h.bootstrap_node_autoid_public_ip + description = "AutoID Bootstrap node IPv4 Addresses" +} + output "domain-node-ipv4-addresses" { value = module.gemini-3h.domain_node_public_ip description = "Domain node IPv4 Addresses" } +output "autoid-node-ipv4-addresses" { + value = module.gemini-3h.autoid_node_public_ip + description = "AutoID node IPv4 Addresses" +} + output "rpc-node-ipv4-addresses" { value = module.gemini-3h.rpc_node_public_ip description = "RPC node IPv4 Addresses" diff --git a/resources/gemini-3h/variables.tf b/resources/gemini-3h/variables.tf index 956275b2..b624016d 100644 --- a/resources/gemini-3h/variables.tf +++ b/resources/gemini-3h/variables.tf @@ -13,14 +13,14 @@ variable "farmer_reward_address" { variable "domain_id" { description = "Domain ID" type = list(number) - default = [0] + default = [0, 1] } //todo change this to a map variable "domain_labels" { description = "Tag of the domain to run" type = list(string) - default = ["evm"] + default = ["nova, "autoid"] } variable "instance_type" { @@ -33,6 +33,7 @@ variable "instance_type" { nova-squid = "c7a.2xlarge" farmer = "c7a.2xlarge" evm_bootstrap = "c7a.xlarge" + autoid_bootstrap = "c7a.xlarge" } } @@ -57,6 +58,7 @@ variable "instance_count" { bootstrap = 2 rpc = 2 domain = 2 + autoid = 2 rpc-squid = 1 nova-squid = 1 farmer = 0 diff --git a/templates/scripts/create_autoid_node_compose_file.sh b/templates/scripts/create_autoid_node_compose_file.sh new file mode 100755 index 00000000..310ba12f --- /dev/null +++ b/templates/scripts/create_autoid_node_compose_file.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +EXTERNAL_IP=`curl -s -4 https://ifconfig.me` +EXTERNAL_IP_V6=`curl -s -6 https://ifconfig.me` + +cat > ~/subspace/docker-compose.yml << EOF +version: "3.7" + +volumes: + archival_node_data: {} + vmagentdata: {} + +networks: + traefik-proxy: + +services: + vmagent: + container_name: vmagent + image: victoriametrics/vmagent:latest + depends_on: + - "archival-node" + ports: + - 8429:8429 + volumes: + - vmagentdata:/vmagentdata + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + command: + - "--httpListenAddr=0.0.0.0:8429" + - "--promscrape.config=/etc/prometheus/prometheus.yml" + - "--remoteWrite.url=http://vmetrics.subspace.network:8428/api/v1/write" + + agent: + container_name: newrelic-infra + image: newrelic/infrastructure:latest + cap_add: + - SYS_PTRACE + network_mode: bridge + pid: host + privileged: true + volumes: + - "/:/host:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + environment: + NRIA_LICENSE_KEY: "\${NR_API_KEY}" + NRIA_DISPLAY_NAME: "\${NETWORK_NAME}-autoid-node-\${NODE_ID}" + restart: unless-stopped + + # traefik reverse proxy with automatic tls management using let encrypt + traefik: + image: traefik:v2.11.3 + container_name: traefik + restart: unless-stopped + command: + - --api=false + - --api.dashboard=false + - --providers.docker + - --log.level=info + - --entrypoints.web.address=:80 + - --entrypoints.web.http.redirections.entryPoint.to=websecure + - --entrypoints.websecure.address=:443 + - --providers.docker=true + - --providers.docker.exposedbydefault=false + - --certificatesresolvers.le.acme.email=alerts@subspace.network + - --certificatesresolvers.le.acme.storage=/acme.json + - --certificatesresolvers.le.acme.tlschallenge=true + - "traefik.docker.network=traefik-proxy" + networks: + - traefik-proxy + ports: + - 80:80 + - 443:443 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./letsencrypt/acme.json:/acme.json + + archival-node: + image: ghcr.io/\${NODE_ORG}/node:\${NODE_TAG} + volumes: + - archival_node_data:/var/subspace:rw + - ./keystore:/var/subspace/keystore:ro + restart: unless-stopped + ports: + - "30333:30333/tcp" + - "30333:30333/udp" + - "30433:30433/tcp" + - "30433:30433/udp" + - "30334:30334/tcp" + - "9615:9615" + labels: + - "traefik.enable=true" + - "traefik.http.services.archival-node.loadbalancer.server.port=8944" + - "traefik.http.routers.archival-node.rule=Host(\`\${DOMAIN_PREFIX_AUTO}-\${DOMAIN_ID_AUTO}.\${NETWORK_NAME}.subspace.network\`) && Path(\`/ws\`)" + - "traefik.http.routers.archival-node.tls=true" + - "traefik.http.routers.archival-node.tls.certresolver=le" + - "traefik.http.routers.archival-node.entrypoints=websecure" + - "traefik.http.routers.archival-node.middlewares=redirect-https" + - "traefik.http.middlewares.redirect-https.redirectscheme.scheme=https" + - "traefik.http.middlewares.redirect-https.redirectscheme.permanent=true" + - "traefik.docker.network=traefik-proxy" + networks: + - traefik-proxy + logging: + driver: loki + options: + loki-url: "https://logging.subspace.network/loki/api/v1/push" + command: [ + "run", + "--chain", "\${NETWORK_NAME}", + "--base-path", "/var/subspace", + "--state-pruning", "archive", + "--blocks-pruning", "archive", + "--pot-external-entropy", "\${POT_EXTERNAL_ENTROPY}", + "--listen-on", "/ip4/0.0.0.0/tcp/30333", + "--listen-on", "/ip6/::/tcp/30333", + "--node-key", "\${NODE_KEY}", + "--in-peers", "500", + "--out-peers", "250", + "--rpc-max-connections", "10000", + "--rpc-cors", "all", + "--rpc-listen-on", "0.0.0.0:9944", + "--rpc-methods", "safe", + "--prometheus-listen-on", "0.0.0.0:9615", +EOF + +reserved_only=${1} +node_count=${2} +current_node=${3} +bootstrap_node_count=${4} +dsn_bootstrap_node_count=${4} +bootstrap_node_autoid_count=${5} +enable_domains=${6} +domain_id=${7} + +for (( i = 0; i < node_count; i++ )); do + if [ "${current_node}" == "${i}" ]; then + dsn_addr=$(sed -nr "s/NODE_${i}_DSN_MULTI_ADDR=//p" ~/subspace/node_keys.txt) + echo " \"--dsn-external-address\", \"${dsn_addr}\"," >> ~/subspace/docker-compose.yml + fi +done + +for (( i = 0; i < bootstrap_node_count; i++ )); do + addr=$(sed -nr "s/NODE_${i}_MULTI_ADDR_TCP=//p" ~/subspace//bootstrap_node_keys.txt) + echo " \"--reserved-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml + echo " \"--bootstrap-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml +done + +for (( i = 0; i < dsn_bootstrap_node_count; i++ )); do + dsn_addr=$(sed -nr "s/NODE_${i}_SUBSPACE_MULTI_ADDR=//p" ~/subspace/dsn_bootstrap_node_keys.txt) + echo " \"--dsn-reserved-peers\", \"${dsn_addr}\"," >> ~/subspace/docker-compose.yml + echo " \"--dsn-bootstrap-nodes\", \"${dsn_addr}\"," >> ~/subspace/docker-compose.yml +done + +if [ "${reserved_only}" == "true" ]; then + echo " \"--reserved-only\"," >> ~/subspace/docker-compose.yml +fi + +if [ "${enable_domains}" == "true" ]; then + { + # auto domain + echo ' "--",' + echo ' "--domain-id", "${DOMAIN_ID_AUTO}",' + echo ' "--state-pruning", "archive",' + echo ' "--blocks-pruning", "archive",' + echo ' "--operator-id", "0",' + echo ' "--listen-on", "/ip4/0.0.0.0/tcp/${OPERATOR_PORT}",' + echo ' "--rpc-cors", "all",' + echo ' "--rpc-listen-on", "0.0.0.0:8944",' + for (( i = 0; i < bootstrap_node_autoid_count; i++ )); do + addr=$(sed -nr "s/NODE_${i}_MULTI_ADDR_TCP=//p" ~/subspace/bootstrap_node_autoid_keys.txt) + echo " \"--reserved-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml + echo " \"--bootstrap-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml + done + } >> ~/subspace/docker-compose.yml +fi + +echo ' ]' >> ~/subspace/docker-compose.yml diff --git a/templates/scripts/create_bootstrap_node_autoid_compose_file.sh b/templates/scripts/create_bootstrap_node_autoid_compose_file.sh new file mode 100755 index 00000000..7bbef0ef --- /dev/null +++ b/templates/scripts/create_bootstrap_node_autoid_compose_file.sh @@ -0,0 +1,197 @@ +#!/bin/bash + +EXTERNAL_IP=`curl -s -4 https://ifconfig.me` +EXTERNAL_IP_V6=`curl -s -6 https://ifconfig.me` + +reserved_only=${1} +node_count=${2} +current_node=${3} +bootstrap_node_count=${4} +enable_domains=${5} + +cat > ~/subspace/docker-compose.yml << EOF +version: "3.7" + +volumes: + archival_node_data: {} + vmagentdata: {} + +services: + vmagent: + container_name: vmagent + image: victoriametrics/vmagent:latest + depends_on: + - "archival-node" + ports: + - 8429:8429 + volumes: + - vmagentdata:/vmagentdata + - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro + command: + - "--httpListenAddr=0.0.0.0:8429" + - "--promscrape.config=/etc/prometheus/prometheus.yml" + - "--remoteWrite.url=http://vmetrics.subspace.network:8428/api/v1/write" + + agent: + container_name: newrelic-infra + image: newrelic/infrastructure:latest + cap_add: + - SYS_PTRACE + network_mode: bridge + pid: host + privileged: true + volumes: + - "/:/host:ro" + - "/var/run/docker.sock:/var/run/docker.sock" + environment: + NRIA_LICENSE_KEY: "\${NR_API_KEY}" + NRIA_DISPLAY_NAME: "\${NETWORK_NAME}-bootstrap-node-autoid-\${NODE_ID}" + restart: unless-stopped + + dsn-bootstrap-node: + image: ghcr.io/\${NODE_ORG}/bootstrap-node:\${NODE_TAG} + restart: unless-stopped + environment: + - RUST_LOG=info + ports: + - "30533:30533/tcp" + - "30533:30533/udp" + - "9616:9616" + logging: + driver: loki + options: + loki-url: "https://logging.subspace.network/loki/api/v1/push" + command: + - start + - "--metrics-endpoints=0.0.0.0:9616" + - "--keypair" + - \${DSN_NODE_KEY} + - "--listen-on" + - /ip4/0.0.0.0/udp/30533/quic-v1 + - "--listen-on" + - /ip4/0.0.0.0/tcp/30533 + - "--listen-on" + - /ip6/::/udp/30533/quic-v1 + - "--listen-on" + - /ip6/::/tcp/30533 + - --protocol-version + - \${GENESIS_HASH} + - "--in-peers" + - "1000" + - "--out-peers" + - "1000" + - "--pending-in-peers" + - "1000" + - "--pending-out-peers" + - "1000" + # - "--external-address" + # - "/ip4/$EXTERNAL_IP/udp/30533/quic-v1" + # - "--external-address" + # - "/ip4/$EXTERNAL_IP/tcp/30533" + # - "--external-address" + # - "/ip6/$EXTERNAL_IP_V6/udp/30533/quic-v1" + # - "--external-address" + # - "/ip6/$EXTERNAL_IP_V6/tcp/30533" +EOF + +for (( i = 0; i < node_count; i++ )); do + if [ "${current_node}" == "${i}" ]; then + dsn_addr=$(sed -nr "s/NODE_${i}_DSN_MULTI_ADDR=//p" ~/subspace/node_keys.txt) + echo " - \"--external-address\"" >> ~/subspace/docker-compose.yml + echo " - \"${dsn_addr}\"" >> ~/subspace/docker-compose.yml + dsn_addr=$(sed -nr "s/NODE_${i}_DSN_MULTI_ADDR_TCP=//p" ~/subspace/node_keys.txt) + echo " - \"--external-address\"" >> ~/subspace/docker-compose.yml + echo " - \"${dsn_addr}\"" >> ~/subspace/docker-compose.yml + fi +done + +for (( i = 0; i < bootstrap_node_count; i++ )); do + dsn_addr=$(sed -nr "s/NODE_${i}_SUBSPACE_MULTI_ADDR=//p" ~/subspace/dsn_bootstrap_node_keys.txt) + echo " - \"--reserved-peers\"" >> ~/subspace/docker-compose.yml + echo " - \"${dsn_addr}\"" >> ~/subspace/docker-compose.yml + echo " - \"--bootstrap-nodes\"" >> ~/subspace/docker-compose.yml + echo " - \"${dsn_addr}\"" >> ~/subspace/docker-compose.yml +done + +cat >> ~/subspace/docker-compose.yml << EOF + archival-node: + image: ghcr.io/\${NODE_ORG}/node:\${NODE_TAG} + volumes: + - archival_node_data:/var/subspace:rw + restart: unless-stopped + ports: + - "30333:30333/udp" + - "30333:30333/tcp" + - "30433:30433/udp" + - "30433:30433/tcp" + - "\${OPERATOR_PORT}:30334/tcp" + - "9615:9615" + logging: + driver: loki + options: + loki-url: "https://logging.subspace.network/loki/api/v1/push" + command: [ + "run", + "--chain", "\${NETWORK_NAME}", + "--base-path", "/var/subspace", + "--state-pruning", "archive", + "--blocks-pruning", "256", + "--pot-external-entropy", "\${POT_EXTERNAL_ENTROPY}", + "--listen-on", "/ip4/0.0.0.0/tcp/30333", + "--listen-on", "/ip6/::/tcp/30333", +## comment to disable external addresses using IP format for now +# "--dsn-external-address", "/ip4/$EXTERNAL_IP/udp/30433/quic-v1", +# "--dsn-external-address", "/ip4/$EXTERNAL_IP/tcp/30433", +# "--dsn-external-address", "/ip6/$EXTERNAL_IP_V6/udp/30433/quic-v1", +# "--dsn-external-address", "/ip6/$EXTERNAL_IP_V6/tcp/30433", + "--node-key", "\${NODE_KEY}", + "--in-peers", "1000", + "--out-peers", "1000", + "--dsn-in-connections", "1000", + "--dsn-out-connections", "1000", + "--dsn-pending-in-connections", "1000", + "--dsn-pending-out-connections", "1000", + "--prometheus-listen-on", "0.0.0.0:9615", +EOF + +for (( i = 0; i < node_count; i++ )); do + if [ "${current_node}" == "${i}" ]; then + dsn_addr=$(sed -nr "s/NODE_${i}_DSN_OPERATOR_MULTI_ADDR=//p" ~/subspace/node_keys.txt) + echo " \"--dsn-external-address\", \"${dsn_addr}\"," >> ~/subspace/docker-compose.yml + fi +done + +for (( i = 0; i < bootstrap_node_count; i++ )); do + addr=$(sed -nr "s/NODE_${i}_MULTI_ADDR_TCP=//p" ~/subspace/bootstrap_node_keys.txt) + echo " \"--reserved-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml + echo " \"--bootstrap-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml +done + +for (( i = 0; i < dsn_bootstrap_node_count; i++ )); do + dsn_addr=$(sed -nr "s/NODE_${i}_SUBSPACE_MULTI_ADDR=//p" ~/subspace/dsn_bootstrap_node_keys.txt) + echo " \"--dsn-reserved-peers\", \"${dsn_addr}\"," >> ~/subspace/docker-compose.yml + echo " \"--dsn-bootstrap-nodes\", \"${dsn_addr}\"," >> ~/subspace/docker-compose.yml +done + +if [ "${reserved_only}" == true ]; then + echo " \"--reserved-only\"," >> ~/subspace/docker-compose.yml +fi + +if [ "${enable_domains}" == "true" ]; then + { + # auto domain + echo ' "--",' + echo ' "--domain-id", "${DOMAIN_ID_AUTO}",' + echo ' "--state-pruning", "archive",' + echo ' "--blocks-pruning", "archive",' + echo ' "--listen-on", "/ip4/0.0.0.0/tcp/${OPERATOR_PORT}",' + echo ' "--rpc-cors", "all",' + echo ' "--rpc-listen-on", "0.0.0.0:8944",' + for (( i = 0; i < node_count; i++ )); do + addr=$(sed -nr "s/NODE_${i}_OPERATOR_MULTI_ADDR_TCP=//p" ~/subspace/node_keys.txt) + echo " \"--reserved-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml + echo " \"--bootstrap-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml + done + } >> ~/subspace/docker-compose.yml +fi +echo ' ]' >> ~/subspace/docker-compose.yml diff --git a/templates/scripts/create_bootstrap_node_evm_compose_file.sh b/templates/scripts/create_bootstrap_node_evm_compose_file.sh index 261d8dc0..ac64e7f6 100755 --- a/templates/scripts/create_bootstrap_node_evm_compose_file.sh +++ b/templates/scripts/create_bootstrap_node_evm_compose_file.sh @@ -45,7 +45,7 @@ services: - "/var/run/docker.sock:/var/run/docker.sock" environment: NRIA_LICENSE_KEY: "\${NR_API_KEY}" - NRIA_DISPLAY_NAME: "\${NETWORK_NAME}-bootstrap-node-evm\${NODE_ID}" + NRIA_DISPLAY_NAME: "\${NETWORK_NAME}-bootstrap-node-evm-\${NODE_ID}" restart: unless-stopped dsn-bootstrap-node: @@ -182,7 +182,7 @@ if [ "${enable_domains}" == "true" ]; then { # core domain echo ' "--",' - echo ' "--domain-id", "${DOMAIN_ID}",' + echo ' "--domain-id", "${DOMAIN_ID_EVM}",' echo ' "--state-pruning", "archive",' echo ' "--blocks-pruning", "archive",' echo ' "--listen-on", "/ip4/0.0.0.0/tcp/${OPERATOR_PORT}",' @@ -193,7 +193,6 @@ if [ "${enable_domains}" == "true" ]; then echo " \"--reserved-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml echo " \"--bootstrap-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml done - } >> ~/subspace/docker-compose.yml fi echo ' ]' >> ~/subspace/docker-compose.yml diff --git a/templates/scripts/create_domain_node_compose_file.sh b/templates/scripts/create_domain_node_compose_file.sh index dbcc5f41..997f9e8c 100755 --- a/templates/scripts/create_domain_node_compose_file.sh +++ b/templates/scripts/create_domain_node_compose_file.sh @@ -89,7 +89,7 @@ services: labels: - "traefik.enable=true" - "traefik.http.services.archival-node.loadbalancer.server.port=8944" - - "traefik.http.routers.archival-node.rule=Host(\`\${DOMAIN_PREFIX}-\${DOMAIN_ID}.\${DOMAIN_LABEL}.\${NETWORK_NAME}.subspace.network\`) && Path(\`/ws\`)" + - "traefik.http.routers.archival-node.rule=Host(\`\${DOMAIN_PREFIX_EVM}-\${DOMAIN_ID_EVM}.\${NETWORK_NAME}.subspace.network\`) && Path(\`/ws\`)" - "traefik.http.routers.archival-node.tls=true" - "traefik.http.routers.archival-node.tls.certresolver=le" - "traefik.http.routers.archival-node.entrypoints=websecure" @@ -159,7 +159,7 @@ if [ "${enable_domains}" == "true" ]; then { # core domain echo ' "--",' - echo ' "--domain-id", "${DOMAIN_ID}",' + echo ' "--domain-id", "${DOMAIN_ID_EVM}",' echo ' "--state-pruning", "archive",' echo ' "--blocks-pruning", "archive",' echo ' "--operator-id", "0",' @@ -174,7 +174,6 @@ if [ "${enable_domains}" == "true" ]; then echo " \"--reserved-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml echo " \"--bootstrap-nodes\", \"${addr}\"," >> ~/subspace/docker-compose.yml done - } >> ~/subspace/docker-compose.yml fi diff --git a/templates/terraform/hetzner/bootstrap_node_evm_provisioner.tf b/templates/terraform/hetzner/bootstrap_node_evm_provisioner.tf index 14bdf170..b36d3dde 100644 --- a/templates/terraform/hetzner/bootstrap_node_evm_provisioner.tf +++ b/templates/terraform/hetzner/bootstrap_node_evm_provisioner.tf @@ -153,10 +153,10 @@ resource "null_resource" "start-bootstrap-nodes-evm" { "echo NETWORK_NAME=${var.network_name} >> /root/subspace/.env", "echo NODE_ID=${count.index} >> /root/subspace/.env", "echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /root/subspace/node_keys.txt) >> /root/subspace/.env", - "echo DOMAIN_LABEL=${var.domain-node-config.domain-labels[0]} >> /root/subspace/.env", - "echo DOMAIN_ID=${var.domain-node-config.domain-id[0]} >> /root/subspace/.env", - "echo RELAYER_SYSTEM_ID=$(sed -nr 's/NODE_${count.index}_RELAYER_SYSTEM_ID=//p' /root/subspace/relayer_ids.txt) >> /root/subspace/.env", - "echo RELAYER_DOMAIN_ID=$(sed -nr 's/NODE_${count.index}_RELAYER_DOMAIN_ID=//p' /root/subspace/relayer_ids.txt) >> /root/subspace/.env", + "echo DOMAIN_LABEL_EVM=${var.domain-node-config.domain-labels[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_EVM=${var.domain-node-config.domain-id[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_LABEL_AUTO=${var.domain-node-config.domain-labels[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_AUTO=${var.domain-node-config.domain-id[1]} >> /home/${var.ssh_user}/subspace/.env", "echo PIECE_CACHE_SIZE=${var.piece_cache_size} >> /root/subspace/.env", "echo DSN_NODE_ID=${count.index} >> /root/subspace/.env", "echo DSN_NODE_KEY=$(sed -nr 's/NODE_${count.index}_DSN_KEY=//p' /root/subspace/node_keys.txt) >> /root/subspace/.env", diff --git a/templates/terraform/hetzner/domain_node_provisioner.tf b/templates/terraform/hetzner/domain_node_provisioner.tf index 5998677f..4d560b2f 100644 --- a/templates/terraform/hetzner/domain_node_provisioner.tf +++ b/templates/terraform/hetzner/domain_node_provisioner.tf @@ -156,14 +156,14 @@ resource "null_resource" "start-domain-nodes" { "echo REPO_ORG=${var.domain-node-config.repo-org} > /root/subspace/.env", "echo NODE_TAG=${var.domain-node-config.node-tag} >> /root/subspace/.env", "echo NETWORK_NAME=${var.network_name} >> /root/subspace/.env", - "echo DOMAIN_PREFIX=${var.domain-node-config.domain-prefix} >> /root/subspace/.env", - # //todo use a map for domain id and labels - "echo DOMAIN_LABEL=${var.domain-node-config.domain-labels[0]} >> /root/subspace/.env", - "echo DOMAIN_ID=${var.domain-node-config.domain-id[0]} >> /root/subspace/.env", + "echo DOMAIN_PREFIX_EVM=${var.domain-node-config.domain-prefix[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_PREFIX_AUTO=${var.domain-node-config.domain-prefix[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_LABEL_EVM=${var.domain-node-config.domain-labels[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_EVM=${var.domain-node-config.domain-id[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_LABEL_AUTO=${var.domain-node-config.domain-labels[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_AUTO=${var.domain-node-config.domain-id[1]} >> /home/${var.ssh_user}/subspace/.env", "echo NODE_ID=${count.index} >> /root/subspace/.env", "echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /root/subspace/node_keys.txt) >> /root/subspace/.env", - "echo RELAYER_SYSTEM_ID=$(sed -nr 's/NODE_${count.index}_RELAYER_SYSTEM_ID=//p' /root/subspace/relayer_ids.txt) >> /root/subspace/.env", - "echo RELAYER_DOMAIN_ID=$(sed -nr 's/NODE_${count.index}_RELAYER_DOMAIN_ID=//p' /root/subspace/relayer_ids.txt) >> /root/subspace/.env", "echo PIECE_CACHE_SIZE=${var.piece_cache_size} >> /root/subspace/.env", "echo NODE_DSN_PORT=${var.domain-node-config.node-dsn-port} >> /root/subspace/.env", "echo BRANCH_NAME=${var.branch_name} >> /root/subspace/.env", diff --git a/templates/terraform/hetzner/variables.tf b/templates/terraform/hetzner/variables.tf index cce6bea7..32a71573 100644 --- a/templates/terraform/hetzner/variables.tf +++ b/templates/terraform/hetzner/variables.tf @@ -36,7 +36,7 @@ variable "domain-node-config" { repo-org = string node-tag = string additional-node-ips = list(string) - domain-prefix = string + domain-prefix = list(string) reserved-only = bool prune = bool node-dsn-port = number diff --git a/templates/terraform/network-primitives/autoid_node_provisioner.tf b/templates/terraform/network-primitives/autoid_node_provisioner.tf new file mode 100644 index 00000000..cb742cd6 --- /dev/null +++ b/templates/terraform/network-primitives/autoid_node_provisioner.tf @@ -0,0 +1,184 @@ +locals { + autoid_nodes_ip_v4 = flatten([ + [aws_instance.autoid_node.*.public_ip] + ] + ) + autoid_nodes_ip_v6 = flatten([ + [aws_instance.autoid_node.*.ipv6_addresses] + ] + ) +} + +resource "null_resource" "setup-autoid-nodes" { + count = length(local.autoid_nodes_ip_v4) + + depends_on = [aws_instance.autoid_node] + + # trigger on node ip changes + triggers = { + cluster_instance_ipv4s = join(",", local.autoid_nodes_ip_v4) + } + + connection { + host = local.autoid_nodes_ip_v4[count.index] + user = var.ssh_user + type = "ssh" + agent = true + private_key = file("${var.private_key_path}") + timeout = "300s" + } + + # create subspace dir + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /home/${var.ssh_user}/subspace/", + "sudo chown -R ${var.ssh_user}:${var.ssh_user} /home/${var.ssh_user}/subspace/ && sudo chmod -R 750 /home/${var.ssh_user}/subspace/" + ] + } + + # copy install file + provisioner "file" { + source = "${var.path_to_scripts}/installer.sh" + destination = "/home/${var.ssh_user}/subspace/installer.sh" + } + + # copy config files + provisioner "file" { + source = "${var.path_to_configs}/" + destination = "/home/${var.ssh_user}/subspace/" + } + + # copy LE script + provisioner "file" { + source = "${var.path_to_scripts}/acme.sh" + destination = "/home/${var.ssh_user}/subspace/acme.sh" + } + + # install docker and docker compose and LE script + provisioner "remote-exec" { + inline = [ + "sudo bash /home/${var.ssh_user}/subspace/installer.sh", + "bash /home/${var.ssh_user}/subspace/acme.sh", + ] + } + +} + +resource "null_resource" "prune-autoid-nodes" { + count = var.autoid-node-config.prune ? length(local.autoid_nodes_ip_v4) : 0 + depends_on = [null_resource.setup-autoid-nodes] + + triggers = { + prune = var.autoid-node-config.prune + } + + connection { + host = local.autoid_nodes_ip_v4[count.index] + user = var.ssh_user + type = "ssh" + agent = true + private_key = file("${var.private_key_path}") + timeout = "300s" + } + + provisioner "file" { + source = "${var.path_to_scripts}/prune_docker_system.sh" + destination = "/home/${var.ssh_user}/subspace/prune_docker_system.sh" + } + + # prune network + provisioner "remote-exec" { + inline = [ + "sudo bash /home/${var.ssh_user}/subspace/prune_docker_system.sh" + ] + } +} + +resource "null_resource" "start-autoid-nodes" { + count = length(local.autoid_nodes_ip_v4) + + depends_on = [null_resource.setup-autoid-nodes] + + # trigger on node deployment version change + triggers = { + deployment_version = var.autoid-node-config.deployment-version + reserved_only = var.autoid-node-config.reserved-only + } + + connection { + host = local.autoid_nodes_ip_v4[count.index] + user = var.ssh_user + type = "ssh" + agent = true + private_key = file("${var.private_key_path}") + timeout = "300s" + } + + # copy node keys file + provisioner "file" { + source = "./autoid_node_keys.txt" + destination = "/home/${var.ssh_user}/subspace/node_keys.txt" + } + + # copy boostrap node keys file + provisioner "file" { + source = "./bootstrap_node_keys.txt" + destination = "/home/${var.ssh_user}/subspace/bootstrap_node_keys.txt" + } + + + # copy boostrap node keys file + provisioner "file" { + source = "./bootstrap_node_autoid_keys.txt" + destination = "/home/${var.ssh_user}/subspace/bootstrap_node_autoid_keys.txt" + } + + # copy dsn_boostrap node keys file + provisioner "file" { + source = "./dsn_bootstrap_node_keys.txt" + destination = "/home/${var.ssh_user}/subspace/dsn_bootstrap_node_keys.txt" + } + + # copy keystore + provisioner "file" { + source = "./keystore" + destination = "/home/${var.ssh_user}/subspace/keystore/" + } + + # copy compose file creation script + provisioner "file" { + source = "${var.path_to_scripts}/create_autoid_node_compose_file.sh" + destination = "/home/${var.ssh_user}/subspace/create_compose_file.sh" + } + + # start docker containers + provisioner "remote-exec" { + inline = [ + # stop any running service + "sudo docker compose -f /home/${var.ssh_user}/subspace/docker-compose.yml down ", + + # set hostname + "sudo hostnamectl set-hostname ${var.network_name}-autoid-node-${count.index}", + + # create .env file + "echo NODE_ORG=${var.autoid-node-config.docker-org} > /home/${var.ssh_user}/subspace/.env", + "echo NODE_TAG=${var.autoid-node-config.docker-tag} >> /home/${var.ssh_user}/subspace/.env", + "echo NETWORK_NAME=${var.network_name} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_PREFIX_AUTO=${var.autoid-node-config.domain-prefix[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_LABEL_AUTO=${var.autoid-node-config.domain-labels[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_AUTO=${var.autoid-node-config.domain-id[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo NODE_ID=${count.index} >> /home/${var.ssh_user}/subspace/.env", + "echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /home/${var.ssh_user}/subspace/node_keys.txt) >> /home/${var.ssh_user}/subspace/.env", + "echo NR_API_KEY=${var.nr_api_key} >> /home/${var.ssh_user}/subspace/.env", + "echo PIECE_CACHE_SIZE=${var.piece_cache_size} >> /home/${var.ssh_user}/subspace/.env", + "echo NODE_DSN_PORT=${var.autoid-node-config.node-dsn-port} >> /home/${var.ssh_user}/subspace/.env", + "echo POT_EXTERNAL_ENTROPY=${var.pot_external_entropy} >> /home/${var.ssh_user}/subspace/.env", + + # create docker compose file + "bash /home/${var.ssh_user}/subspace/create_compose_file.sh ${var.bootstrap-node-config.reserved-only} ${length(local.domain_nodes_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${length(local.bootstrap_nodes_autoid_ip_v4)} ${var.autoid-node-config.enable-domains} ${var.autoid-node-config.domain-id[0]}", + + # start subspace node + "sudo docker compose -f /home/${var.ssh_user}/subspace/docker-compose.yml up -d", + ] + } +} diff --git a/templates/terraform/network-primitives/bootstrap_node_autoid_provisioner.tf b/templates/terraform/network-primitives/bootstrap_node_autoid_provisioner.tf new file mode 100644 index 00000000..5d557b5b --- /dev/null +++ b/templates/terraform/network-primitives/bootstrap_node_autoid_provisioner.tf @@ -0,0 +1,168 @@ +locals { + bootstrap_nodes_autoid_ip_v4 = flatten([ + [aws_instance.bootstrap_node_autoid.*.public_ip] + ] + ) + + bootstrap_nodes_autoid_ip_v6 = flatten([ + [aws_instance.bootstrap_node_autoid.*.ipv6_addresses] + ] + ) +} + +resource "null_resource" "setup-bootstrap-nodes-autoid" { + count = length(local.bootstrap_nodes_autoid_ip_v4) + + depends_on = [aws_instance.bootstrap_node_autoid] + + # trigger on node ip changes + triggers = { + cluster_instance_ipv4s = join(",", local.bootstrap_nodes_autoid_ip_v4) + } + + connection { + host = local.bootstrap_nodes_autoid_ip_v4[count.index] + user = var.ssh_user + type = "ssh" + agent = true + private_key = file("${var.private_key_path}") + timeout = "300s" + } + + # create subspace dir + provisioner "remote-exec" { + inline = [ + "sudo mkdir -p /home/${var.ssh_user}/subspace/", + "sudo chown -R ${var.ssh_user}:${var.ssh_user} /home/${var.ssh_user}/subspace/ && sudo chmod -R 750 /home/${var.ssh_user}/subspace/" + ] + } + + # copy install file + provisioner "file" { + source = "${var.path_to_scripts}/installer.sh" + destination = "/home/${var.ssh_user}/subspace/installer.sh" + } + + # copy config files + provisioner "file" { + source = "${var.path_to_configs}/" + destination = "/home/${var.ssh_user}/subspace/" + } + + # install docker and docker compose + provisioner "remote-exec" { + inline = [ + "sudo bash /home/${var.ssh_user}/subspace/installer.sh", + ] + } + +} + +resource "null_resource" "prune-bootstrap-nodes-autoid" { + count = var.bootstrap-node-autoid-config.prune ? length(local.bootstrap_nodes_autoid_ip_v4) : 0 + depends_on = [null_resource.setup-bootstrap-nodes-autoid] + + triggers = { + prune = var.bootstrap-node-autoid-config.prune + } + + connection { + host = local.bootstrap_nodes_autoid_ip_v4[count.index] + user = var.ssh_user + type = "ssh" + agent = true + private_key = file("${var.private_key_path}") + timeout = "300s" + } + + provisioner "file" { + source = "${var.path_to_scripts}/prune_docker_system.sh" + destination = "/home/${var.ssh_user}/subspace/prune_docker_system.sh" + } + + # prune network + provisioner "remote-exec" { + inline = [ + "sudo bash /home/${var.ssh_user}/subspace/prune_docker_system.sh" + ] + } +} + +resource "null_resource" "start-bootstrap-nodes-autoid" { + count = length(local.bootstrap_nodes_autoid_ip_v4) + + depends_on = [null_resource.setup-bootstrap-nodes-autoid] + + # trigger on node deployment version change + triggers = { + deployment_version = var.bootstrap-node-autoid-config.deployment-version + reserved_only = var.bootstrap-node-autoid-config.reserved-only + } + + connection { + host = local.bootstrap_nodes_autoid_ip_v4[count.index] + user = var.ssh_user + type = "ssh" + agent = true + private_key = file("${var.private_key_path}") + timeout = "300s" + } + + # copy bootstrap node keys file + provisioner "file" { + source = "./bootstrap_node_autoid_keys.txt" + destination = "/home/${var.ssh_user}/subspace/node_keys.txt" + } + + # copy boostrap node keys file + provisioner "file" { + source = "./bootstrap_node_keys.txt" + destination = "/home/${var.ssh_user}/subspace/bootstrap_node_keys.txt" + } + + # copy DSN bootstrap node keys file + provisioner "file" { + source = "./dsn_bootstrap_node_keys.txt" + destination = "/home/${var.ssh_user}/subspace/dsn_bootstrap_node_keys.txt" + } + + # copy compose file creation script + provisioner "file" { + source = "${var.path_to_scripts}/create_bootstrap_node_autoid_compose_file.sh" + destination = "/home/${var.ssh_user}/subspace/create_compose_file.sh" + } + + # start docker containers + provisioner "remote-exec" { + inline = [ + # stop any running service + "sudo docker compose -f /home/${var.ssh_user}/subspace/docker-compose.yml down ", + + # set hostname + "sudo hostnamectl set-hostname ${var.network_name}-bootstrap-node-autoid-${count.index}", + + # create .env file + "echo NODE_ORG=${var.bootstrap-node-autoid-config.docker-org} > /home/${var.ssh_user}/subspace/.env", + "echo NODE_TAG=${var.bootstrap-node-autoid-config.docker-tag} >> /home/${var.ssh_user}/subspace/.env", + "echo NETWORK_NAME=${var.network_name} >> /home/${var.ssh_user}/subspace/.env", + "echo NODE_ID=${count.index} >> /home/${var.ssh_user}/subspace/.env", + "echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /home/${var.ssh_user}/subspace/node_keys.txt) >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_LABEL_AUTO=${var.domain-node-config.domain-labels[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_AUTO=${var.domain-node-config.domain-id[1]} >> /home/${var.ssh_user}/subspace/.env", + "echo NR_API_KEY=${var.nr_api_key} >> /home/${var.ssh_user}/subspace/.env", + "echo PIECE_CACHE_SIZE=${var.piece_cache_size} >> /home/${var.ssh_user}/subspace/.env", + "echo DSN_NODE_ID=${count.index} >> /home/${var.ssh_user}/subspace/.env", + "echo DSN_NODE_KEY=$(sed -nr 's/NODE_${count.index}_DSN_KEY=//p' /home/${var.ssh_user}/subspace/node_keys.txt) >> /home/${var.ssh_user}/subspace/.env", + "echo DSN_LISTEN_PORT=${var.bootstrap-node-autoid-config.dsn-listen-port} >> /home/${var.ssh_user}/subspace/.env", + "echo NODE_DSN_PORT=${var.bootstrap-node-autoid-config.node-dsn-port} >> /home/${var.ssh_user}/subspace/.env", + "echo OPERATOR_PORT=${var.bootstrap-node-autoid-config.operator-port} >> /home/${var.ssh_user}/subspace/.env", + "echo GENESIS_HASH=${var.bootstrap-node-autoid-config.genesis-hash} >> /home/${var.ssh_user}/subspace/.env", + + # create docker compose file + "bash /home/${var.ssh_user}/subspace/create_compose_file.sh ${var.bootstrap-node-autoid-config.reserved-only} ${length(local.bootstrap_nodes_autoid_ip_v4)} ${count.index} ${length(local.bootstrap_nodes_ip_v4)} ${var.domain-node-config.enable-domains} ", + + # start subspace node + "sudo docker compose -f /home/${var.ssh_user}/subspace/docker-compose.yml up -d", + ] + } +} diff --git a/templates/terraform/network-primitives/dns.tf b/templates/terraform/network-primitives/dns.tf index 87a7a520..c65dab7c 100644 --- a/templates/terraform/network-primitives/dns.tf +++ b/templates/terraform/network-primitives/dns.tf @@ -10,6 +10,21 @@ resource "cloudflare_record" "rpc" { type = "A" } +resource "cloudflare_record" "nova" { + count = length(local.domain_nodes_ip_v4) + zone_id = data.cloudflare_zone.cloudflare_zone.id + name = "${var.domain-node-config.domain-prefix[0]}-${var.domain-node-config.domain-id[0]}.${var.network_name}" + value = local.domain_nodes_ip_v4[count.index] + type = "A" +} + +resource "cloudflare_record" "nova_ipv6" { + count = length(local.domain_nodes_ip_v4) + zone_id = data.cloudflare_zone.cloudflare_zone.id + name = "${var.domain-node-config.domain-prefix[0]}-${var.domain-node-config.domain-id[0]}.${var.network_name}" + value = local.domain_nodes_ip_v6[count.index] + type = "AAAA" + resource "cloudflare_record" "rpc-squid" { count = length(local.rpc_squid_nodes_ip_v4) zone_id = data.cloudflare_zone.cloudflare_zone.id @@ -26,23 +41,22 @@ resource "cloudflare_record" "nova-squid-rpc" { type = "A" } -resource "cloudflare_record" "nova" { +resource "cloudflare_record" "auto" { count = length(local.domain_nodes_ip_v4) zone_id = data.cloudflare_zone.cloudflare_zone.id - name = "${var.domain-node-config.domain-prefix}-${count.index}.${var.network_name}" - value = local.domain_nodes_ip_v6[count.index] + name = "${var.autoid-node-config.domain-prefix[1]}-${var.autoid-node-config.domain-id[1]}.${var.network_name}" + value = local.domain_nodes_ip_v4[count.index] type = "A" } -resource "cloudflare_record" "nova_ipv6" { +resource "cloudflare_record" "auto_ipv6" { count = length(local.domain_nodes_ip_v4) zone_id = data.cloudflare_zone.cloudflare_zone.id - name = "${var.domain-node-config.domain-prefix}-${count.index}.${var.network_name}" + name = "${var.autoid-node-config.domain-prefix[1]}-${var.autoid-node-config.domain-id[1]}.${var.network_name}" value = local.domain_nodes_ip_v6[count.index] type = "AAAA" } - resource "cloudflare_record" "bootstrap" { count = length(local.bootstrap_nodes_ip_v4) zone_id = data.cloudflare_zone.cloudflare_zone.id @@ -74,3 +88,19 @@ resource "cloudflare_record" "bootstrap_evm_ipv6" { value = local.bootstrap_nodes_evm_ip_v6[count.index] type = "AAAA" } + +resource "cloudflare_record" "bootstrap_auto" { + count = length(local.bootstrap_nodes_autoid_ip_v4) + zone_id = data.cloudflare_zone.cloudflare_zone.id + name = "bootstrap-${count.index}.auto.${var.network_name}" + value = local.bootstrap_nodes_autoid_ip_v6[count.index] + type = "A" +} + +resource "cloudflare_record" "bootstrap_auto_ipv6" { + count = length(local.bootstrap_nodes_autoid_ip_v4) + zone_id = data.cloudflare_zone.cloudflare_zone.id + name = "bootstrap-${count.index}.auto.${var.network_name}" + value = local.bootstrap_nodes_autoid_ip_v6[count.index] + type = "AAAA" +} diff --git a/templates/terraform/network-primitives/domain_node_provisioner.tf b/templates/terraform/network-primitives/domain_node_provisioner.tf index ed0688a1..39d0f766 100644 --- a/templates/terraform/network-primitives/domain_node_provisioner.tf +++ b/templates/terraform/network-primitives/domain_node_provisioner.tf @@ -164,10 +164,9 @@ resource "null_resource" "start-domain-nodes" { "echo NODE_ORG=${var.domain-node-config.docker-org} > /home/${var.ssh_user}/subspace/.env", "echo NODE_TAG=${var.domain-node-config.docker-tag} >> /home/${var.ssh_user}/subspace/.env", "echo NETWORK_NAME=${var.network_name} >> /home/${var.ssh_user}/subspace/.env", - "echo DOMAIN_PREFIX=${var.domain-node-config.domain-prefix} >> /home/${var.ssh_user}/subspace/.env", - # //todo use a map for domain id and labels - "echo DOMAIN_LABEL=${var.domain-node-config.domain-labels[0]} >> /home/${var.ssh_user}/subspace/.env", - "echo DOMAIN_ID=${var.domain-node-config.domain-id[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_PREFIX_EVM=${var.domain-node-config.domain-prefix[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_LABEL_EVM=${var.domain-node-config.domain-labels[0]} >> /home/${var.ssh_user}/subspace/.env", + "echo DOMAIN_ID_EVM=${var.domain-node-config.domain-id[0]} >> /home/${var.ssh_user}/subspace/.env", "echo NODE_ID=${count.index} >> /home/${var.ssh_user}/subspace/.env", "echo NODE_KEY=$(sed -nr 's/NODE_${count.index}_KEY=//p' /home/${var.ssh_user}/subspace/node_keys.txt) >> /home/${var.ssh_user}/subspace/.env", "echo NR_API_KEY=${var.nr_api_key} >> /home/${var.ssh_user}/subspace/.env", diff --git a/templates/terraform/network-primitives/instances.tf b/templates/terraform/network-primitives/instances.tf index 4818799e..9f18fa28 100644 --- a/templates/terraform/network-primitives/instances.tf +++ b/templates/terraform/network-primitives/instances.tf @@ -126,6 +126,70 @@ resource "aws_instance" "bootstrap_node_evm" { } +resource "aws_instance" "bootstrap_node_autoid" { + count = length(var.aws_region) * var.bootstrap-node-autoid-config.instance-count + ami = data.aws_ami.ubuntu_amd64.image_id + instance_type = var.bootstrap-node-autoid-config.instance-type + subnet_id = element(aws_subnet.public_subnets.*.id, 0) + availability_zone = var.azs + ipv6_address_count = 1 + # Security Group + vpc_security_group_ids = ["${aws_security_group.network_sg.id}"] + # the Public SSH key + key_name = var.aws_key_name + associate_public_ip_address = true + ebs_optimized = true + ebs_block_device { + device_name = "/dev/sda1" + volume_size = var.bootstrap-node-config.disk-volume-size + volume_type = var.bootstrap-node-config.disk-volume-type + iops = 3000 + throughput = 250 + } + + + tags = { + Name = "${var.network_name}-bootstrap-autoid-${count.index}" + name = "${var.network_name}-bootstrap-autoid-${count.index}" + role = "bootstrap node" + os_name = "ubuntu" + os_version = "22.04" + arch = "x86_64" + } + + depends_on = [ + aws_subnet.public_subnets, + #aws_nat_gateway.nat_gateway, + aws_internet_gateway.gw + ] + + lifecycle { + + ignore_changes = [ami, ipv6_address_count] + + } + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait", + "sudo apt update -y", + ] + + on_failure = continue + + } + + # Setting up the ssh connection + connection { + type = "ssh" + host = element(self.*.public_ip, count.index) + user = var.ssh_user + private_key = file("${var.private_key_path}") + timeout = "300s" + } + +} + resource "aws_instance" "rpc_squid_node" { count = length(var.aws_region) * var.rpc-squid-node-config.instance-count ami = data.aws_ami.ubuntu_amd64.image_id @@ -379,6 +443,68 @@ resource "aws_instance" "domain_node" { } +resource "aws_instance" "autoid_node" { + count = length(var.aws_region) * var.autoid-node-config.instance-count + ami = data.aws_ami.ubuntu_amd64.image_id + instance_type = var.autoid-node-config.instance-type + subnet_id = element(aws_subnet.public_subnets.*.id, 0) + availability_zone = var.azs + ipv6_address_count = 1 + # Security Group + vpc_security_group_ids = ["${aws_security_group.network_sg.id}"] + # the Public SSH key + key_name = var.aws_key_name + associate_public_ip_address = true + ebs_optimized = true + ebs_block_device { + device_name = "/dev/sda1" + volume_size = var.autoid-node-config.disk-volume-size + volume_type = var.autoid-node-config.disk-volume-type + iops = 3000 + throughput = 250 + } + + tags = { + Name = "${var.network_name}-autoid-${count.index}" + name = "${var.network_name}-autoid-${count.index}" + role = "autoid node" + os_name = "ubuntu" + os_version = "22.04" + arch = "x86_64" + } + + depends_on = [ + aws_subnet.public_subnets, + #aws_nat_gateway.nat_gateway, + aws_internet_gateway.gw + ] + + lifecycle { + + ignore_changes = [ami, ipv6_address_count] + + } + + provisioner "remote-exec" { + inline = [ + "cloud-init status --wait", + "sudo apt update -y", + ] + + on_failure = continue + + } + + # Setting up the ssh connection + connection { + type = "ssh" + host = element(self.*.public_ip, count.index) + user = var.ssh_user + private_key = file("${var.private_key_path}") + timeout = "300s" + } + +} resource "aws_instance" "farmer_node" { count = length(var.aws_region) * var.farmer-node-config.instance-count diff --git a/templates/terraform/network-primitives/outputs.tf b/templates/terraform/network-primitives/outputs.tf index 246dc7c6..b0343c50 100644 --- a/templates/terraform/network-primitives/outputs.tf +++ b/templates/terraform/network-primitives/outputs.tf @@ -32,6 +32,22 @@ output "bootstrap_node_evm_ami" { value = aws_instance.bootstrap_node_evm.*.ami } +output "bootstrap_node_autoid_server_id" { + value = aws_instance.bootstrap_node_autoid.*.id +} + +output "bootstrap_node_autoid_public_ip" { + value = aws_instance.bootstrap_node_autoid.*.public_ip +} + +output "bootstrap_node_autoid_private_ip" { + value = aws_instance.bootstrap_node_autoid.*.private_ip +} + +output "bootstrap_node_autoid_ami" { + value = aws_instance.bootstrap_node_autoid.*.ami +} + output "rpc_squid_node_server_id" { value = aws_instance.rpc_squid_node.*.id } @@ -98,6 +114,22 @@ output "domain_node_ami" { value = aws_instance.domain_node.*.ami } +output "autoid_node_server_id" { + value = aws_instance.autoid_node.*.id +} + +output "autoid_node_private_ip" { + value = aws_instance.autoid_node.*.private_ip +} + +output "autoid_node_public_ip" { + value = aws_instance.autoid_node.*.public_ip +} + +output "autoid_node_ami" { + value = aws_instance.autoid_node.*.ami +} + output "farmer_node_server_id" { value = aws_instance.farmer_node.*.id @@ -122,5 +154,6 @@ output "dns-records" { cloudflare_record.rpc-squid.*.hostname, cloudflare_record.nova-squid-rpc.*.hostname, cloudflare_record.nova.*.hostname, + cloudflare_record.autoid.*.hostname, ] } diff --git a/templates/terraform/network-primitives/variables.tf b/templates/terraform/network-primitives/variables.tf index a66566ee..ee260c69 100644 --- a/templates/terraform/network-primitives/variables.tf +++ b/templates/terraform/network-primitives/variables.tf @@ -40,6 +40,7 @@ variable "instance_count" { bootstrap = 2 rpc = 2 domain = 2 + autoid = 2 rpc-squid = 1 nova-squid = 1 farmer = 1 @@ -161,7 +162,28 @@ variable "domain-node-config" { instance-count = number docker-org = string docker-tag = string - domain-prefix = string + domain-prefix = list(string) + reserved-only = bool + prune = bool + node-dsn-port = number + enable-domains = bool + domain-id = list(number) + domain-labels = list(string) + disk-volume-size = number + disk-volume-type = string + }) +} + +variable "autoid-node-config" { + description = "AutoID node deployment config" + type = object({ + instance-type = string + deployment-version = number + regions = list(string) + instance-count = number + docker-org = string + docker-tag = string + domain-prefix = list(string) reserved-only = bool prune = bool node-dsn-port = number @@ -233,6 +255,26 @@ variable "bootstrap-node-evm-config" { }) } +variable "bootstrap-node-autoid-config" { + description = "Bootstrap node autoid domain deployment config" + type = object({ + instance-type = string + deployment-version = number + regions = list(string) + instance-count = number + docker-org = string + docker-tag = string + reserved-only = bool + prune = bool + genesis-hash = string + dsn-listen-port = number + node-dsn-port = number + operator-port = number + disk-volume-size = number + disk-volume-type = string + }) +} + variable "farmer-node-config" { description = "Farmer and Node configuration" type = object({