From 6f8ccd0c96bc6ae7a9f6ad964e6882da3654d593 Mon Sep 17 00:00:00 2001 From: Greg Szabo <16846635+greg-szabo@users.noreply.github.com> Date: Wed, 12 Jun 2024 15:43:24 -0400 Subject: [PATCH] feat(qa): DigitalOcean infrastructure (#222) * feat(metrics): Add basic metrics for consensus and associated configuration * Add `malachite_consensus_time_per_block` metric * Cleanup * QA infra * doc update * Track remaining metrics * Export libp2p and GossipSub metrics using a prefixed registry - Gossip consensus metrics are prefixed with `malachite_gossip_consensus_` - Gossip mempool metrics are prefixed with `malachite_gossip_mempool_` * Pass metrics registry around instead of relying on global one * Formatting * Fix prefix * Fix span * Add `metrics.enabled` config option * Fix spelling * Cleanup * First stab at a local data viewer * env.var update * db export update * qa improvements * grafana exports * fix typo * fix merge * more commands * commands tweaking * simplify commands.sh * deploy tests * README update --------- Co-authored-by: Romain Ruetschi Co-authored-by: Anca Zamfir --- code/.dockerignore | 2 + qa/.gitignore | 10 + qa/README.md | 253 +++++++++ qa/docker/Dockerfile | 13 + qa/terraform/file-commands.tf | 35 ++ qa/terraform/file-hosts.tf | 30 + qa/terraform/nodes.tf | 59 ++ qa/terraform/project.tf | 5 + qa/terraform/provider.tf | 14 + qa/terraform/templates/commands.tmpl | 264 +++++++++ qa/terraform/templates/hosts.tmpl | 14 + qa/terraform/user-data/cc-data.txt | 417 ++++++++++++++ qa/terraform/user-data/user-data.txt | 165 ++++++ qa/terraform/variables.tf | 22 + qa/viewer/Makefile | 25 + qa/viewer/compose.yml | 45 ++ .../provisioning/dashboards-data/main.json | 536 ++++++++++++++++++ .../provisioning/dashboards/malachite.yml | 13 + .../provisioning/datasources/prometheus.yml | 9 + qa/viewer/config-prometheus/prometheus.yml | 210 +++++++ 20 files changed, 2141 insertions(+) create mode 100644 code/.dockerignore create mode 100644 qa/.gitignore create mode 100644 qa/README.md create mode 100644 qa/docker/Dockerfile create mode 100644 qa/terraform/file-commands.tf create mode 100644 qa/terraform/file-hosts.tf create mode 100644 qa/terraform/nodes.tf create mode 100644 qa/terraform/project.tf create mode 100644 qa/terraform/provider.tf create mode 100644 qa/terraform/templates/commands.tmpl create mode 100644 qa/terraform/templates/hosts.tmpl create mode 100644 qa/terraform/user-data/cc-data.txt create mode 100644 qa/terraform/user-data/user-data.txt create mode 100644 qa/terraform/variables.tf create mode 100644 qa/viewer/Makefile create mode 100644 qa/viewer/compose.yml create mode 100644 qa/viewer/config-grafana/provisioning/dashboards-data/main.json create mode 100644 qa/viewer/config-grafana/provisioning/dashboards/malachite.yml create mode 100644 qa/viewer/config-grafana/provisioning/datasources/prometheus.yml create mode 100644 qa/viewer/config-prometheus/prometheus.yml diff --git a/code/.dockerignore b/code/.dockerignore new file mode 100644 index 000000000..39a8ce060 --- /dev/null +++ b/code/.dockerignore @@ -0,0 +1,2 @@ +target +scripts diff --git a/qa/.gitignore b/qa/.gitignore new file mode 100644 index 000000000..cd585698e --- /dev/null +++ b/qa/.gitignore @@ -0,0 +1,10 @@ +terraform/.terraform +terraform/.terraform.lock.hcl +terraform/.terraform.tfstate.lock.info +terraform/terraform.tfvars +terraform/terraform.tfstate* +terraform/hosts +terraform/commands.sh +viewer/data-grafana +viewer/data-prometheus +viewer/prometheus.tgz diff --git a/qa/README.md b/qa/README.md new file mode 100644 index 000000000..76b7d160f --- /dev/null +++ b/qa/README.md @@ -0,0 +1,253 @@ +# QA + +This is an opinionated QA environment with a human developer in mind. It focuses on logical blocks of a QA setup +using custom commands to simplify the language used to describe the process of running the nodes. + +## Prerequisites + +* [pssh](https://linux.die.net/man/1/pssh)(Mac) or [parallel-ssh](https://manpages.org/parallel-ssh)(Linux) on your + local machine. +* If you use parallel-ssh, create a symlink to `pssh` in your path. + +* Usually, `ln /usr/bin/parallel-ssh /usr/bin/pssh` will do the trick. + +## The command & control server + +A `cc` server is deployed along with the QA nodes. It helps manage the servers, and it is closer than a developer +machine. + +The developer can build the Docker image for testing locally and push it to the Docker Registry on the `cc` server, +using the `deploy_cc` custom command. The QA nodes can then pull the image from the registry and run it. + +The developer can create the testnet configuration remotely on the `cc` server using the `setup_config` custom command. +The configuration is stored in the `/data` folder on the server which is shared as over NFS with the QA nodes. + +The `cc` server also hosts a Prometheus server with Grafana for monitoring the nodes. The data can be downloaded using +the `get_prometheus_data` custom command. Then it can be imported to a local Grafana/Prometheus viewer for further +analysis. + +Finally, the `cc` server also works as the DNS server for the QA nodes. All node IPs can be resolved by simple names on +the servers. This is especially useful when configuring persistent peers. + +## Set up the hosts in Digital Ocean + +After creating your DO access (see the CometBFT QA infra +[steps](https://github.com/cometbft/qa-infra/blob/main/README.md#setup)), run + +```bash +cd terraform +terraform init +terraform apply -var small_nodes=0 # optional. This will create the cc server only. +terraform apply -var small_nodes=4 -var large_nodes=3 # the cc server will not be deleted if you scale the nodes. +``` + +By running terraform with zero nodes first, you create the `cc` server ahead of time. You can skip that step and create +the `cc` server with the QA nodes in one go. + +The above will create a 7-node Digital Ocean QA environment a `hosts` file and a `commands.sh` file with the custom +commands. + +Most of the node setup is done automatically in cloud-init. When terraform finishes, the servers are still installing +packages and setting up their environment. One of the first commands we will run will check if the servers have +finished building. + +## Post-terraform tasks + +There are a few custom commands to make managing the nodes easier. They are explained in the `commands.sh` file. + +Note: most of these commands require SSH authentication. If you use a Yubikey for SSH authentication, you can +saturate your machine's SSH connection with the default settings. Use a key file and `ssh-agent` or change +connection settings. + +### 0. TL;DR + +You start execution on your local machine and move over to the `cc` server when it is ready. You can also keep working +from your local machine if you feel the servers are close enough and the network is fast. + +```bash +source commands.sh # do this in all new terminal window on your machine. No need to do this on the CC server. + +ok_cc # make sure the CC server has finished initial setup. +deploy_cc # Takes 4-5 minutes. Continue in a different window while this is running. + # You can run it on cc server as well, but you have to manually put the source code at /root/malachite. + +ssh-cc # (optional) move to the CC server and run the rest of the commands closer to the QA nodes. +setup_config # depends on deploy_cc, only run it if that finished. + +ok_all # make sure all QA servers have finished initial setup +dnode-run all # run malachite on all QA servers + +# Wait some time to generate data + +dnode-stop all # stop all malachite nodes. It does not remove the docker container so the logs can be viewed. + +get_prometheus_data # this has to run on the machine where you want the data to end up. Usually, your local machine. +fetch_log all # fetch he logs of malachite-cli from each QA node + +dnode-rm all # remove the docker container "node" from the servers so the application can be re-run +``` + +### 1. Import custom commands + +```bash +source commands.sh +``` + +Make the custom commands available on your local machine. You do not need to run this on the CC server, as it gets +invoked automatically when you SSH into the server. + +### 2. Make sure CC works + +```bash +ok_cc +``` + +This loads the SSH key into your known_hosts and checks if the cloud-init execution has finished on the CC server. It +also sets up the DNS service with the created hosts and copies the `commands.sh` over for easy execution. + +It will print a date if the server successfully finished the setup. + +You have to run this every time you create or destroy new servers with Terraform. It copies the server IPs and the +correct custom commands to the CC server. + +### 4. Build your node and deploy it to the cc server. + +```bash +deploy_cc +``` + +Builds the application using Docker and deploys it into the CC server Docker Registry. + +This will take a few minutes. (4.5 minutes in Lausanne, connecting to a 4vCPU/8GB fra1 server in Digital Ocean.) + +You can continue executing the rest of the setup commands, until you want to configure the network with `setup_config`. +You will need the application for the correct generation of the application configuration. + +You can also run this command on the `cc` server (see the `ssh-cc` command below). Caveat: you need to copy the source +code over to the server + +### 4.5 (optional) Connect to the CC server + +```bash +ssh-cc +``` + +It is encouraged to run the rest of the commands from the CC server as it is closer to the QA servers and the commands +run faster. + +The custom commands are automatically available on the CC server. No need to `source commands.sh` there. + +You can keep running on your local machine, though, if that is more convenient. + +### 5. Make sure all servers finished cloud-init installations + +```bash +ok_all +``` + +Similar to `ok_cc` but all deployed servers are taken into account. Your `known_hosts` file will be updated with the +server keys and prints the date each server finished installing cloud-init. Run this multiple times until all servers +return successfully. + +### 6. Create the configuration data on the cc server + +```bash +setup_config +``` + +The configuration data is stored on the CC server under `/data`. This path is also shared with the QA nodes over NFS. + +Depends on an up-to-date host count. Re-run it after `ok_cc` if you changed the number of servers. + +### 7. Start the nodes + +```bash +dnode-run 0 2 3 +RUST_LOG=debug cnode-run 1 +``` + +You can also use the `all` keyword to start or stop all nodes at once. + +```bash +dnode-stop all +``` + +You can use `dnode`, `dnode-run`, `dnode-log` and `dnode-stop` to manage the docker container. +`dnode` is a generic command to run docker commands remotely. + +### 8. Get the data from Prometheus + +```bash +get_prometheus_data +``` + +This will copy the compressed prometheus database from the `cc` server to your local machine as `prometheus.tgz`. + +# Created files + +## hosts file + +Terraform creates a [hosts](terraform/hosts) file that can be added to any server (including your local dev machine) +for easier access to the servers. The file is +deployed onto the cc server and it is used as part of the DNS service there. + +## commands.sh file + +Terraform also creates a [commands.sh](terraform/commands.sh) file with suggested commands for CLI-based configuration +and node +management. You can run `source commands.sh` and use the functions in your shell. The descriptions of commands are +listed in the top comment of the file. The file is copied over to `cc` during `ok_cc` and invoked automatically +when you SSH into the server. + +## prometheus.tgz file + +This file gets exported using the `get_prometheus_data` command. Import it in the viewer for further analysis. + +# Viewer + +The viewer allows you to view the metrics of a testnet on your local machine. You can export the Prometheus metrics +from the cloud and keep them on your local machine even after the testnet is destroyed. You can do additional analysis +and create custom Grafana dashboards. + +## Prerequisites + +* docker on your machine +* a running `cc` server from where you download the data +* `make` on your machine. + +The commands that start with `make` will need to be run from the `viewer` directory or you can use `-C` to point make +to the directory. + +## 1. Download the data + +This command is part of the terraform-created `commands.sh` file. + +```bash +download_data +``` + +This will download compressed `prometheus.tgz` file from the `cc` server. + +## 2. Extract the data to its destination + +```bash +make extract_data +``` + +You can give a different path to the command if you stored the file elsewhere with the `FILE` environment variable. + +## 3. Start the viewer + +```bash +make viewer-start +``` + +You can view the Grafana dashboard at `http://localhost:3000`. The default username and password are `admin`/`admin`. + +## 4. Finish up + +When you are done with the data, you can stop the viewer. + +```bash +make viewer-stop +``` diff --git a/qa/docker/Dockerfile b/qa/docker/Dockerfile new file mode 100644 index 000000000..45531602e --- /dev/null +++ b/qa/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM rust AS builder +ARG PROTOC_VERSION +ENV PROTOC_VERSION=${PROTOC_VERSION:-27.0} +RUN --mount=type=bind,from=code,target=/mnt < /dev/null + PSSH_P=1 PSSH_H=$CANDC xssh "cat /etc/done" && \ + echo "Updating cc server..." && \ + scp -q "$${1:-$${MALACHITE_DIR}/qa/terraform/hosts}" root@$${CANDC}:/etc/hosts && \ + ssh root@$${CANDC} "systemctl restart dnsmasq" && \ + scp -q "$${1:-$${MALACHITE_DIR}/qa/terraform/commands.sh}" root@$${CANDC}:/etc/profile.d/commands.sh && \ + ssh root@$${CANDC} \ + "sed -i 's,^export MALACHITE_DIR=.*,export MALACHITE_DIR=/root/malachite,' /etc/profile.d/commands.sh && \ + sed -i 's,^export IS_CC=.*,export IS_CC=1,' /etc/profile.d/commands.sh && \ + source /etc/profile.d/commands.sh && \ + _keyscan_all_servers 2> /dev/null" +} + +ok_all() { + _keyscan_all_servers 2> /dev/null + xssh "cat /etc/done && mount /data" # Mount /data in case a QA node came online earlier than CC +} + +deploy_cc() { + test -d $MALACHITE_DIR/code || (echo "Source code repository not found. Clone or copy manually." && return 1) + docker $_CC_DOCKER_SHIM build --push -t cc.testnet/node --build-context code=$MALACHITE_DIR/code $MALACHITE_DIR/qa/docker +} + +setup_config() { + docker $_CC_DOCKER_SHIM run -d --pull always --dns ${cc.internal_ip} -v /data:/data -e RUST_LOG ${cc.internal_ip}/node --home /data testnet --nodes $D_N --deterministic + if _is_cc; then + _change_config all + else + ssh root@$CANDC "source /etc/profile.d/commands.sh && _change_config all" + fi +} + +dnode-pull() { + PSSH_H="$(_parse_multiple_hosts "$@")" xssh docker pull ${cc.internal_ip}/node +} + +dnode-run() { + PSSH_H="$(_parse_multiple_hosts "$@")" xssh docker run -d -p 27000:27000/udp -p 28000:28000/udp -p 9000:9000/tcp --name node --cap-add=NET_ADMIN --dns ${cc.internal_ip} -v /data:/data -v /config:/config -e RUST_LOG ${cc.internal_ip}/node --home /config start +} + +dnode-log() { + IP="$(get_ip "$1")" + F="" + if [ "$${1:-}" = "-f" ]; then + F="-f" + IP="$(get_ip "$2")" + else + if [ "$${2:-}" = "-f" ]; then + F="-f" + fi + fi + docker -H ssh://root@$IP logs $F node +} + +dnode-stop() { + PSSH_H="$(_parse_multiple_hosts "$@")" xssh docker stop node +} + +dnode-rm() { + PSSH_H="$(_parse_multiple_hosts "$@")" xssh "docker stop node 2> /dev/null; docker rm node" +} + +cheat_sheet() { +cat < /dev/null && tar -cvzf prometheus.tgz -C /var/lib/prometheus/metrics2 . ; systemctl start prometheus + else + ssh-cc "systemctl stop prometheus && rm prometheus.tgz 2> /dev/null && tar -cvzf prometheus.tgz -C /var/lib/prometheus/metrics2 . ; systemctl start prometheus" + scp -r root@$CANDC:prometheus.tgz . + fi +} + +mem_usage() { + PSSH_H="$(_parse_multiple_hosts "$@")" xssh -o mem_usage_out -e mem_usage_err "ps -e -o pid,user,%mem,cmd --sort=-%mem | head -2 | tail -1" +} + +cpu_usage() { + PSSH_H="$(_parse_multiple_hosts "$@")" xssh -o cpu_usage_out -e cpu_usage_err "ps -e -o pid,user,%cpu,cmd --sort=-%cpu | head -2 | tail -1" +} + +reset_prometheus_db() { + if _is_cc; then + systemctl stop prometheus + rm -rf /var/lib/prometheus/metrics2/* + systemctl start prometheus + else + ssh-cc "systemctl stop prometheus && rm -rf /var/lib/prometheus/metrics2/*; systemctl start prometheus" + fi +} + +_is_cc() { + return $IS_CC +} + +_keyscan_cc() { + ssh-keygen -R $CANDC > /dev/null + ssh-keyscan -t ed25519 $CANDC >> $HOME/.ssh/known_hosts +} + +_keyscan_all_servers() { + _keyscan_cc 2> /dev/null +%{~ for n in concat(small, large) } + ssh-keygen -R ${n.ip} > /dev/null + ssh-keyscan -t ed25519 ${n.ip} >> $HOME/.ssh/known_hosts 2> /dev/null +%{~ endfor } +} + +_deploy_sync_code() { + rsync -avz --exclude target $MALACHITE_DIR/code root@$${CANDC}:/root/malachite/ +} + +_deploy_sync_qa() { + rsync -avz --exclude terraform --exclude viewer $MALACHITE_DIR/qa root@$${CANDC}:/root/malachite/ +} + +_deploy_build() { + if _is_cc; then + docker build --push -t cc.testnet/node --build-context code=$MALACHITE_DIR/code $MALACHITE_DIR/qa/docker + else + ssh-cc docker build --push -t cc.testnet/node --build-context code=$MALACHITE_DIR/code $MALACHITE_DIR/qa/docker + fi +} + +_compose_persistent_peers() { + port=$${1:-27000} + + persistent_peers="" + for i in $(seq 0 ${length(small)-1}) + do + persistent_peers="$persistent_peers,/dns/small$i/udp/$port/quic-v1" + done + for i in $(seq 0 ${length(large)-1}) + do + persistent_peers="$persistent_peers,/dns/large$i/udp/$port/quic-v1" + done + echo $${persistent_peers##,} +} + +_change_config() { + P="$@" + if [ "$P" = "all" ]; then + P="$(seq 0 $((D_N-1)))" + fi + for i in $P + do + file="/data/$i/config/config.toml" + sconfig "$file" \ + "moniker=test-$i" \ + "consensus.p2p.listen_addr=/ip4/0.0.0.0/udp/27000/quic-v1" \ + "mempool.p2p.listen_addr=/ip4/0.0.0.0/udp/28000/quic-v1" \ + "metrics.listen_addr=0.0.0.0:9000" \ + "test.time_allowance_factor=0.5" \ + "test.exec_time_per_tx=500us" && \ + sconfig "$file" -t stringSlice \ + "consensus.p2p.persistent_peers=$(_compose_persistent_peers)" \ + "mempool.p2p.persistent_peers=$(_compose_persistent_peers 28000)" & + done +} + +_change_one_config_entry() { + P="$(seq 0 $((D_N-1)))" + for i in $P + do + file="/data/$i/config/config.toml" + sconfig "$file" "$@" + done +} + +_parse_multiple_hosts() { + PSSH_X="" + if [ "$1" = "all" ] || [ $# -eq 0 ]; then + PSSH_X="$PSSH_H" + else + while (( "$#" )); + do + PSSH_X="$PSSH_X $(get_ip "$1")" + shift + done + fi + echo "$PSSH_X" +} diff --git a/qa/terraform/templates/hosts.tmpl b/qa/terraform/templates/hosts.tmpl new file mode 100644 index 000000000..bee6d7a9e --- /dev/null +++ b/qa/terraform/templates/hosts.tmpl @@ -0,0 +1,14 @@ +${cc.ip} g-${cc.name} +%{~ for n in small } +${n.ip} g-${n.name} +%{~ endfor } +%{~ for n in large } +${n.ip} g-${n.name} +%{~ endfor } +${cc.internal_ip} ${cc.name} +%{~ for n in small } +${n.internal_ip} ${n.name} +%{~ endfor } +%{~ for n in large } +${n.internal_ip} ${n.name} +%{~ endfor } diff --git a/qa/terraform/user-data/cc-data.txt b/qa/terraform/user-data/cc-data.txt new file mode 100644 index 000000000..fd071762d --- /dev/null +++ b/qa/terraform/user-data/cc-data.txt @@ -0,0 +1,417 @@ +#cloud-config +manage_etc_hosts: false +apt: + sources: + source1: + source: "deb https://download.docker.com/linux/debian $RELEASE stable" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- +package_update: true +packages: + - git + - gcc + - prometheus + - prometheus-node-exporter + - ntpstat + - jq + - ufw + - tmux + - apt-transport-https + - ca-certificates + - curl + - gnupg-agent + - software-properties-common + - docker-ce + - docker-ce-cli + - containerd.io + - nfs-kernel-server + - dnsmasq + - pssh + - rsync +write_files: + - path: /etc/docker/daemon.json + content: | + { + "insecure-registries" : ["0.0.0.0/0"] + } + - path: /etc/systemd/resolved.conf.d/DigitalOcean.conf + content: | + [Resolve] + DNS=127.0.0.1 + DNSStubListener=no + - path: /etc/exports + content: | + /data 10.0.0.0/8(ro,sync,no_subtree_check) + /data 172.16.0.0/12(ro,sync,no_subtree_check) + /data 192.168.0.0/16(ro,sync,no_subtree_check) + - path: /etc/dnsmasq.d/servers.conf + content: | + server=1.0.0.1 + server=1.1.1.1 + bogus-priv + expand-hosts + domain=testnet + cache-size=1000 + - path: /etc/prometheus/prometheus.yml + content: | + global: + scrape_interval: 5s + evaluation_interval: 10s + scrape_configs: + - job_name: 'qa' + static_configs: + - targets: [ + 'small0:9000', + 'small1:9000', + 'small2:9000', + 'small3:9000', + 'small4:9000', + 'small5:9000', + 'small6:9000', + 'small7:9000', + 'small8:9000', + 'small9:9000', + 'small10:9000', + 'small11:9000', + 'small12:9000', + 'small13:9000', + 'small14:9000', + 'small15:9000', + 'small16:9000', + 'small17:9000', + 'small18:9000', + 'small19:9000', + 'small20:9000', + 'small21:9000', + 'small22:9000', + 'small23:9000', + 'small24:9000', + 'small25:9000', + 'small26:9000', + 'small27:9000', + 'small28:9000', + 'small29:9000', + 'small30:9000', + 'small31:9000', + 'small32:9000', + 'small33:9000', + 'small34:9000', + 'small35:9000', + 'small36:9000', + 'small37:9000', + 'small38:9000', + 'small39:9000', + 'small40:9000', + 'small41:9000', + 'small42:9000', + 'small43:9000', + 'small44:9000', + 'small45:9000', + 'small46:9000', + 'small47:9000', + 'small48:9000', + 'small49:9000', + 'small50:9000', + 'small51:9000', + 'small52:9000', + 'small53:9000', + 'small54:9000', + 'small55:9000', + 'small56:9000', + 'small57:9000', + 'small58:9000', + 'small59:9000', + 'small60:9000', + 'small61:9000', + 'small62:9000', + 'small63:9000', + 'small64:9000', + 'small65:9000', + 'small66:9000', + 'small67:9000', + 'small68:9000', + 'small69:9000', + 'small70:9000', + 'small71:9000', + 'small72:9000', + 'small73:9000', + 'small74:9000', + 'small75:9000', + 'small76:9000', + 'small77:9000', + 'small78:9000', + 'small79:9000', + 'small80:9000', + 'small81:9000', + 'small82:9000', + 'small83:9000', + 'small84:9000', + 'small85:9000', + 'small86:9000', + 'small87:9000', + 'small88:9000', + 'small89:9000', + 'small90:9000', + 'small91:9000', + 'small92:9000', + 'small93:9000', + 'small94:9000', + 'small95:9000', + 'small96:9000', + 'small97:9000', + 'small98:9000', + 'small99:9000', + 'large0:9000', + 'large1:9000', + 'large2:9000', + 'large3:9000', + 'large4:9000', + 'large5:9000', + 'large6:9000', + 'large7:9000', + 'large8:9000', + 'large9:9000', + 'large10:9000', + 'large11:9000', + 'large12:9000', + 'large13:9000', + 'large14:9000', + 'large15:9000', + 'large16:9000', + 'large17:9000', + 'large18:9000', + 'large19:9000', + 'large20:9000', + 'large21:9000', + 'large22:9000', + 'large23:9000', + 'large24:9000', + 'large25:9000', + 'large26:9000', + 'large27:9000', + 'large28:9000', + 'large29:9000', + 'large30:9000', + 'large31:9000', + 'large32:9000', + 'large33:9000', + 'large34:9000', + 'large35:9000', + 'large36:9000', + 'large37:9000', + 'large38:9000', + 'large39:9000', + 'large40:9000', + 'large41:9000', + 'large42:9000', + 'large43:9000', + 'large44:9000', + 'large45:9000', + 'large46:9000', + 'large47:9000', + 'large48:9000', + 'large49:9000', + 'large50:9000', + 'large51:9000', + 'large52:9000', + 'large53:9000', + 'large54:9000', + 'large55:9000', + 'large56:9000', + 'large57:9000', + 'large58:9000', + 'large59:9000', + 'large60:9000', + 'large61:9000', + 'large62:9000', + 'large63:9000', + 'large64:9000', + 'large65:9000', + 'large66:9000', + 'large67:9000', + 'large68:9000', + 'large69:9000', + 'large70:9000', + 'large71:9000', + 'large72:9000', + 'large73:9000', + 'large74:9000', + 'large75:9000', + 'large76:9000', + 'large77:9000', + 'large78:9000', + 'large79:9000', + 'large80:9000', + 'large81:9000', + 'large82:9000', + 'large83:9000', + 'large84:9000', + 'large85:9000', + 'large86:9000', + 'large87:9000', + 'large88:9000', + 'large89:9000', + 'large90:9000', + 'large91:9000', + 'large92:9000', + 'large93:9000', + 'large94:9000', + 'large95:9000', + 'large96:9000', + 'large97:9000', + 'large98:9000', + 'large99:9000' + ] + scrape_interval: 1s + - path: /root/grafana.yml + content: | + apiVersion: 1 + datasources: + - name: prometheus + uid: prometheus + type: prometheus + url: http://host.docker.internal:9090 + is_default: true + editable: true + - path: /etc/nsswitch.conf + content: | + passwd: files + group: files + shadow: files + gshadow: files + hosts: dns + networks: files + protocols: db files + services: db files + ethers: db files + rpc: db files + netgroup: nis + - path: /root/compose.yml + content: | + services: + registry: + container_name: registry + image: registry:2 + ports: + - 0.0.0.0:80:5000 + volumes: + - registry:/var/lib/registry + environment: + REGISTRY_HTTP_SECRET: siSRSRTHSRTHSERGehrgjsoiejrg45625623452345isejrgisejrgsergserg + restart: on-failure + grafana: + container_name: grafana + image: grafana/grafana-oss + volumes: + - /root/grafana.yml:/etc/grafana/provisioning/datasources/prometheus.yml + - grafana:/var/lib/grafana + ports: + - 0.0.0.0:3000:3000 + environment: + GF_SECURITY_ADMIN_USER: testnet + GF_SECURITY_ADMIN_PASSWORD: militant-souvenir-dash-teleview + GF_LOG_LEVEL: error + GF_ANALYTICS_ENABLED: false + GF_ANALYTICS_REPORTING_ENABLED: false + GF_ANALYTICS_CHECK_FOR_PLUGIN_UPDATES: false + GF_ANALYTICS_CHECK_FOR_UPDATES: false + GF_ANALYTICS_FEEDBACK_LINKS_ENABLED: false + GF_SECURITY_DISABLE_GRAVATAR: true + GF_USERS_DEFAULT_THEME: system + GF_USERS_EDITORS_CAN_ADMIN: true + GF_AUTH_ANONYMOUS_ENABLED: true + GF_AUTH_ANONYMOUS_ORG_ROLE: Editor + GF_AUTH_BASIC_ENABLED: false + GF_NEWS_NEWS_FEED_ENABLED: false + GF_RENDERING_RENDERER_TOKEN: "-" + GF_RENDERING_SERVER_URL: http://grafana-image-renderer:8081/render + GF_RENDERING_CALLBACK_URL: http://grafana:3000/ + GF_LOG_FILTERS: rendering:debug + extra_hosts: + - "host.docker.internal:host-gateway" + grafana-image-renderer: + image: grafana/grafana-image-renderer + container_name: grafana-image-renderer + volumes: + registry: + grafana: +runcmd: + - ln /usr/bin/parallel-ssh /usr/bin/pssh + - mkdir /data + - chown nobody:nogroup /data + - systemctl daemon-reload + - systemctl restart systemd-resolved + - systemctl enable prometheus-node-exporter + - systemctl start prometheus-node-exporter + - systemctl enable docker + - systemctl start docker + - systemctl enable nfs-kernel-server + - systemctl start nfs-kernel-server + - systemctl restart systemd-journald + - docker compose -f /root/compose.yml up -d + - curl -s -o /usr/bin/sconfig -L https://github.com/freshautomations/sconfig/releases/download/v0.2.0/sconfig_linux_amd64 + - chmod 755 /usr/bin/sconfig + - date > /etc/done + - ln /etc/done /etc/cc + diff --git a/qa/terraform/user-data/user-data.txt b/qa/terraform/user-data/user-data.txt new file mode 100644 index 000000000..671b00b52 --- /dev/null +++ b/qa/terraform/user-data/user-data.txt @@ -0,0 +1,165 @@ +#cloud-config +manage_etc_hosts: false +network: + version: 1 + config: + - type: nameserver + address: + - ${cc.internal_ip} + search: + - testnet +apt: + sources: + source1: + source: "deb https://download.docker.com/linux/debian $RELEASE stable" + key: | + -----BEGIN PGP PUBLIC KEY BLOCK----- + + mQINBFit2ioBEADhWpZ8/wvZ6hUTiXOwQHXMAlaFHcPH9hAtr4F1y2+OYdbtMuth + lqqwp028AqyY+PRfVMtSYMbjuQuu5byyKR01BbqYhuS3jtqQmljZ/bJvXqnmiVXh + 38UuLa+z077PxyxQhu5BbqntTPQMfiyqEiU+BKbq2WmANUKQf+1AmZY/IruOXbnq + L4C1+gJ8vfmXQt99npCaxEjaNRVYfOS8QcixNzHUYnb6emjlANyEVlZzeqo7XKl7 + UrwV5inawTSzWNvtjEjj4nJL8NsLwscpLPQUhTQ+7BbQXAwAmeHCUTQIvvWXqw0N + cmhh4HgeQscQHYgOJjjDVfoY5MucvglbIgCqfzAHW9jxmRL4qbMZj+b1XoePEtht + ku4bIQN1X5P07fNWzlgaRL5Z4POXDDZTlIQ/El58j9kp4bnWRCJW0lya+f8ocodo + vZZ+Doi+fy4D5ZGrL4XEcIQP/Lv5uFyf+kQtl/94VFYVJOleAv8W92KdgDkhTcTD + G7c0tIkVEKNUq48b3aQ64NOZQW7fVjfoKwEZdOqPE72Pa45jrZzvUFxSpdiNk2tZ + XYukHjlxxEgBdC/J3cMMNRE1F4NCA3ApfV1Y7/hTeOnmDuDYwr9/obA8t016Yljj + q5rdkywPf4JF8mXUW5eCN1vAFHxeg9ZWemhBtQmGxXnw9M+z6hWwc6ahmwARAQAB + tCtEb2NrZXIgUmVsZWFzZSAoQ0UgZGViKSA8ZG9ja2VyQGRvY2tlci5jb20+iQI3 + BBMBCgAhBQJYrefAAhsvBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEI2BgDwO + v82IsskP/iQZo68flDQmNvn8X5XTd6RRaUH33kXYXquT6NkHJciS7E2gTJmqvMqd + tI4mNYHCSEYxI5qrcYV5YqX9P6+Ko+vozo4nseUQLPH/ATQ4qL0Zok+1jkag3Lgk + jonyUf9bwtWxFp05HC3GMHPhhcUSexCxQLQvnFWXD2sWLKivHp2fT8QbRGeZ+d3m + 6fqcd5Fu7pxsqm0EUDK5NL+nPIgYhN+auTrhgzhK1CShfGccM/wfRlei9Utz6p9P + XRKIlWnXtT4qNGZNTN0tR+NLG/6Bqd8OYBaFAUcue/w1VW6JQ2VGYZHnZu9S8LMc + FYBa5Ig9PxwGQOgq6RDKDbV+PqTQT5EFMeR1mrjckk4DQJjbxeMZbiNMG5kGECA8 + g383P3elhn03WGbEEa4MNc3Z4+7c236QI3xWJfNPdUbXRaAwhy/6rTSFbzwKB0Jm + ebwzQfwjQY6f55MiI/RqDCyuPj3r3jyVRkK86pQKBAJwFHyqj9KaKXMZjfVnowLh + 9svIGfNbGHpucATqREvUHuQbNnqkCx8VVhtYkhDb9fEP2xBu5VvHbR+3nfVhMut5 + G34Ct5RS7Jt6LIfFdtcn8CaSas/l1HbiGeRgc70X/9aYx/V/CEJv0lIe8gP6uDoW + FPIZ7d6vH+Vro6xuWEGiuMaiznap2KhZmpkgfupyFmplh0s6knymuQINBFit2ioB + EADneL9S9m4vhU3blaRjVUUyJ7b/qTjcSylvCH5XUE6R2k+ckEZjfAMZPLpO+/tF + M2JIJMD4SifKuS3xck9KtZGCufGmcwiLQRzeHF7vJUKrLD5RTkNi23ydvWZgPjtx + Q+DTT1Zcn7BrQFY6FgnRoUVIxwtdw1bMY/89rsFgS5wwuMESd3Q2RYgb7EOFOpnu + w6da7WakWf4IhnF5nsNYGDVaIHzpiqCl+uTbf1epCjrOlIzkZ3Z3Yk5CM/TiFzPk + z2lLz89cpD8U+NtCsfagWWfjd2U3jDapgH+7nQnCEWpROtzaKHG6lA3pXdix5zG8 + eRc6/0IbUSWvfjKxLLPfNeCS2pCL3IeEI5nothEEYdQH6szpLog79xB9dVnJyKJb + VfxXnseoYqVrRz2VVbUI5Blwm6B40E3eGVfUQWiux54DspyVMMk41Mx7QJ3iynIa + 1N4ZAqVMAEruyXTRTxc9XW0tYhDMA/1GYvz0EmFpm8LzTHA6sFVtPm/ZlNCX6P1X + zJwrv7DSQKD6GGlBQUX+OeEJ8tTkkf8QTJSPUdh8P8YxDFS5EOGAvhhpMBYD42kQ + pqXjEC+XcycTvGI7impgv9PDY1RCC1zkBjKPa120rNhv/hkVk/YhuGoajoHyy4h7 + ZQopdcMtpN2dgmhEegny9JCSwxfQmQ0zK0g7m6SHiKMwjwARAQABiQQ+BBgBCAAJ + BQJYrdoqAhsCAikJEI2BgDwOv82IwV0gBBkBCAAGBQJYrdoqAAoJEH6gqcPyc/zY + 1WAP/2wJ+R0gE6qsce3rjaIz58PJmc8goKrir5hnElWhPgbq7cYIsW5qiFyLhkdp + YcMmhD9mRiPpQn6Ya2w3e3B8zfIVKipbMBnke/ytZ9M7qHmDCcjoiSmwEXN3wKYI + mD9VHONsl/CG1rU9Isw1jtB5g1YxuBA7M/m36XN6x2u+NtNMDB9P56yc4gfsZVES + KA9v+yY2/l45L8d/WUkUi0YXomn6hyBGI7JrBLq0CX37GEYP6O9rrKipfz73XfO7 + JIGzOKZlljb/D9RX/g7nRbCn+3EtH7xnk+TK/50euEKw8SMUg147sJTcpQmv6UzZ + cM4JgL0HbHVCojV4C/plELwMddALOFeYQzTif6sMRPf+3DSj8frbInjChC3yOLy0 + 6br92KFom17EIj2CAcoeq7UPhi2oouYBwPxh5ytdehJkoo+sN7RIWua6P2WSmon5 + U888cSylXC0+ADFdgLX9K2zrDVYUG1vo8CX0vzxFBaHwN6Px26fhIT1/hYUHQR1z + VfNDcyQmXqkOnZvvoMfz/Q0s9BhFJ/zU6AgQbIZE/hm1spsfgvtsD1frZfygXJ9f + irP+MSAI80xHSf91qSRZOj4Pl3ZJNbq4yYxv0b1pkMqeGdjdCYhLU+LZ4wbQmpCk + SVe2prlLureigXtmZfkqevRz7FrIZiu9ky8wnCAPwC7/zmS18rgP/17bOtL4/iIz + QhxAAoAMWVrGyJivSkjhSGx1uCojsWfsTAm11P7jsruIL61ZzMUVE2aM3Pmj5G+W + 9AcZ58Em+1WsVnAXdUR//bMmhyr8wL/G1YO1V3JEJTRdxsSxdYa4deGBBY/Adpsw + 24jxhOJR+lsJpqIUeb999+R8euDhRHG9eFO7DRu6weatUJ6suupoDTRWtr/4yGqe + dKxV3qQhNLSnaAzqW/1nA3iUB4k7kCaKZxhdhDbClf9P37qaRW467BLCVO/coL3y + Vm50dwdrNtKpMBh3ZpbB1uJvgi9mXtyBOMJ3v8RZeDzFiG8HdCtg9RvIt/AIFoHR + H3S+U79NT6i0KPzLImDfs8T7RlpyuMc4Ufs8ggyg9v3Ae6cN3eQyxcK3w0cbBwsh + /nQNfsA6uu+9H7NhbehBMhYnpNZyrHzCmzyXkauwRAqoCbGCNykTRwsur9gS41TQ + M8ssD1jFheOJf3hODnkKU+HKjvMROl1DK7zdmLdNzA1cvtZH/nCC9KPj1z8QC47S + xx+dTZSx4ONAhwbS/LN3PoKtn8LPjY9NP9uDWI+TWYquS2U+KHDrBDlsgozDbs/O + jCxcpDzNmXpWQHEtHU7649OXHP7UeNST1mCUCH5qdank0V1iejF6/CfTFU4MfcrG + YT90qFF93M3v01BbxP+EIY2/9tiIPbrd + =0YYh + -----END PGP PUBLIC KEY BLOCK----- +package_update: true +packages: + - git + - gcc + - golang-1.21-go + - prometheus + - prometheus-node-exporter + - ntpstat + - jq + - ufw + - tmux + - apt-transport-https + - ca-certificates + - curl + - gnupg-agent + - software-properties-common + - docker-ce + - docker-ce-cli + - containerd.io + - nfs-common +write_files: +# Prometheus node exporter service + - path: /etc/prometheus/prometheus-node-exporter.service + content: | + [Unit] + Description=Node Exporter + Wants=network-online.target + After=network-online.target + [Service] + User=prometheus + Group=prometheus + Type=simple + ExecStart=/usr/bin/prometheus-node-exporter + [Install] + WantedBy=multi-user.target +# Allow docker to download from insecure registries + - path: /etc/docker/daemon.json + content: | + { + "insecure-registries" : ["${cc.internal_ip}"] + } +# Force systemd-resolved to use custom DNS server + - path: /etc/systemd/resolved.conf.d/DigitalOcean.conf + content: | + [Resolve] + DNS=${cc.internal_ip} +# Force resolv to use DNS for host resolution + - path: /etc/nsswitch.conf + content: | + passwd: files + group: files + shadow: files + gshadow: files + hosts: dns + networks: files + protocols: db files + services: db files + ethers: db files + rpc: db files + netgroup: nis +# Set server ID + - path: /etc/id + content: | + ${id} +runcmd: +# Integrate Prometheus node exporter service + - systemctl daemon-reload +# Update resolv.conf with forced DNS server + - systemctl restart systemd-resolved +# Start prometheus node exporter + - systemctl enable prometheus-node-exporter + - systemctl start prometheus-node-exporter +# Start docker + - systemctl enable docker + - systemctl start docker +# Fix journald not logging entries + - systemctl restart systemd-journald +# Set up config directory + - ln -s /data/${id} /config +# Set up NFS share + - mkdir /data + - chown nobody:nogroup /data + - echo "${cc.internal_ip}:/data /data nfs defaults 0 0" >> /etc/fstab +# Mount NFS share + - sleep 60 # Wait for CC NFS Server to come up with fingers crossed + - mount /data +# Indicate finish + - date > /etc/done diff --git a/qa/terraform/variables.tf b/qa/terraform/variables.tf new file mode 100644 index 000000000..32e468a26 --- /dev/null +++ b/qa/terraform/variables.tf @@ -0,0 +1,22 @@ +variable "small_nodes" { + type = number + default = 2 +} + +variable "large_nodes" { + type = number + default = 0 +} + +variable "region" { + type = string + default = "fra1" +} + +output "next_steps" { + value = <