From 0ed854422ba2f8699c80a7c28f0cc085e764581b Mon Sep 17 00:00:00 2001 From: Javier Cortejoso Date: Thu, 6 Jul 2023 10:27:14 +0200 Subject: [PATCH] Remove terraform (internal) (#10413) * Removed terraform (non-public) and celotool commands --- .env | 2 - .env.alfajores | 2 - .env.baklava | 1 - .env.oracledev | 1 - .env.rc1staging | 3 - .env.staging | 1 - .../src/cmds/deploy/destroy/cluster.ts | 4 +- .../celotool/src/cmds/deploy/destroy/forno.ts | 14 - .../src/cmds/deploy/destroy/testnet.ts | 2 - .../src/cmds/deploy/destroy/vm-testnet.ts | 16 - .../celotool/src/cmds/deploy/initial/forno.ts | 14 - .../src/cmds/deploy/initial/testnet.ts | 3 - .../src/cmds/deploy/initial/vm-testnet.ts | 41 -- .../celotool/src/cmds/deploy/upgrade/forno.ts | 14 - .../src/cmds/deploy/upgrade/testnet.ts | 3 - .../src/cmds/deploy/upgrade/vm-testnet.ts | 48 -- packages/celotool/src/cmds/ssh-vm-node.ts | 51 -- packages/celotool/src/cmds/vm-exec.ts | 185 ------ packages/celotool/src/lib/cluster.ts | 8 +- packages/celotool/src/lib/env-utils.ts | 23 +- packages/celotool/src/lib/forno.ts | 112 ---- packages/celotool/src/lib/geth.ts | 29 +- packages/celotool/src/lib/helm_deploy.ts | 12 +- packages/celotool/src/lib/mock-oracle.ts | 7 +- packages/celotool/src/lib/port_forward.ts | 15 +- .../src/lib/transaction-metrics-exporter.ts | 6 +- packages/celotool/src/lib/utils.ts | 10 +- packages/celotool/src/lib/vm-testnet-utils.ts | 559 ------------------ packages/terraform-modules/.gitignore | 1 - packages/terraform-modules/README.md | 20 - packages/terraform-modules/forno/main.tf | 176 ------ .../forno/modules/backends/main.tf | 46 -- .../forno/modules/backends/outputs.tf | 3 - .../forno/modules/backends/variables.tf | 46 -- packages/terraform-modules/forno/outputs.tf | 3 - .../forno/security_policy.tf | 27 - packages/terraform-modules/forno/variables.tf | 69 --- packages/terraform-modules/forno/versions.tf | 3 - .../testnet-network/README.md | 13 - .../terraform-modules/testnet-network/main.tf | 25 - .../testnet-network/variables.tf | 19 - packages/terraform-modules/testnet/README.md | 41 -- packages/terraform-modules/testnet/main.tf | 242 -------- .../testnet/modules/bootnode/main.tf | 54 -- .../testnet/modules/bootnode/outputs.tf | 3 - .../testnet/modules/bootnode/startup.sh | 149 ----- .../testnet/modules/bootnode/variables.tf | 39 -- .../testnet/modules/full-node/main.tf | 116 ---- .../testnet/modules/full-node/outputs.tf | 19 - .../testnet/modules/full-node/startup.sh | 241 -------- .../testnet/modules/full-node/variables.tf | 139 ----- .../modules/tx-node-load-balancer/main.tf | 244 -------- .../modules/tx-node-load-balancer/output.tf | 3 - .../tx-node-load-balancer/ssl-startup.sh | 129 ---- .../tx-node-load-balancer/variables.tf | 49 -- .../testnet/modules/validator/main.tf | 191 ------ .../testnet/modules/validator/outputs.tf | 7 - .../testnet/modules/validator/startup.sh | 300 ---------- .../testnet/modules/validator/variables.tf | 94 --- packages/terraform-modules/testnet/outputs.tf | 23 - .../terraform-modules/testnet/variables.tf | 144 ----- 61 files changed, 33 insertions(+), 3831 deletions(-) delete mode 100644 packages/celotool/src/cmds/deploy/destroy/forno.ts delete mode 100644 packages/celotool/src/cmds/deploy/destroy/vm-testnet.ts delete mode 100644 packages/celotool/src/cmds/deploy/initial/forno.ts delete mode 100644 packages/celotool/src/cmds/deploy/initial/vm-testnet.ts delete mode 100644 packages/celotool/src/cmds/deploy/upgrade/forno.ts delete mode 100644 packages/celotool/src/cmds/deploy/upgrade/vm-testnet.ts delete mode 100644 packages/celotool/src/cmds/ssh-vm-node.ts delete mode 100644 packages/celotool/src/cmds/vm-exec.ts delete mode 100644 packages/celotool/src/lib/forno.ts delete mode 100644 packages/celotool/src/lib/vm-testnet-utils.ts delete mode 100644 packages/terraform-modules/.gitignore delete mode 100644 packages/terraform-modules/README.md delete mode 100644 packages/terraform-modules/forno/main.tf delete mode 100644 packages/terraform-modules/forno/modules/backends/main.tf delete mode 100644 packages/terraform-modules/forno/modules/backends/outputs.tf delete mode 100644 packages/terraform-modules/forno/modules/backends/variables.tf delete mode 100644 packages/terraform-modules/forno/outputs.tf delete mode 100644 packages/terraform-modules/forno/security_policy.tf delete mode 100644 packages/terraform-modules/forno/variables.tf delete mode 100644 packages/terraform-modules/forno/versions.tf delete mode 100644 packages/terraform-modules/testnet-network/README.md delete mode 100644 packages/terraform-modules/testnet-network/main.tf delete mode 100644 packages/terraform-modules/testnet-network/variables.tf delete mode 100644 packages/terraform-modules/testnet/README.md delete mode 100644 packages/terraform-modules/testnet/main.tf delete mode 100644 packages/terraform-modules/testnet/modules/bootnode/main.tf delete mode 100644 packages/terraform-modules/testnet/modules/bootnode/outputs.tf delete mode 100644 packages/terraform-modules/testnet/modules/bootnode/startup.sh delete mode 100644 packages/terraform-modules/testnet/modules/bootnode/variables.tf delete mode 100644 packages/terraform-modules/testnet/modules/full-node/main.tf delete mode 100644 packages/terraform-modules/testnet/modules/full-node/outputs.tf delete mode 100644 packages/terraform-modules/testnet/modules/full-node/startup.sh delete mode 100644 packages/terraform-modules/testnet/modules/full-node/variables.tf delete mode 100644 packages/terraform-modules/testnet/modules/tx-node-load-balancer/main.tf delete mode 100644 packages/terraform-modules/testnet/modules/tx-node-load-balancer/output.tf delete mode 100644 packages/terraform-modules/testnet/modules/tx-node-load-balancer/ssl-startup.sh delete mode 100644 packages/terraform-modules/testnet/modules/tx-node-load-balancer/variables.tf delete mode 100644 packages/terraform-modules/testnet/modules/validator/main.tf delete mode 100644 packages/terraform-modules/testnet/modules/validator/outputs.tf delete mode 100644 packages/terraform-modules/testnet/modules/validator/startup.sh delete mode 100644 packages/terraform-modules/testnet/modules/validator/variables.tf delete mode 100644 packages/terraform-modules/testnet/outputs.tf delete mode 100644 packages/terraform-modules/testnet/variables.tf diff --git a/.env b/.env index 8087faab907..a2fa43ce4d1 100644 --- a/.env +++ b/.env @@ -8,8 +8,6 @@ GETH_VMODULE="consensus/*=2" GETH_ENABLE_METRICS=false GETH_USE_MYCELO=false -VM_BASED=false - KUBERNETES_CLUSTER_NAME="celo-networks-dev" KUBERNETES_CLUSTER_ZONE="us-west1-a" CLUSTER_DOMAIN_NAME="celo-networks-dev" diff --git a/.env.alfajores b/.env.alfajores index a59e37cff62..f6762b6487e 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -2,8 +2,6 @@ ENV_TYPE="production" GETH_VERBOSITY=2 -VM_BASED=false - KUBERNETES_CLUSTER_NAME="alfajores" KUBERNETES_CLUSTER_ZONE="us-west1-a" CLUSTER_DOMAIN_NAME="celo-testnet" diff --git a/.env.baklava b/.env.baklava index d353c7b4db6..4262cee6c2f 100644 --- a/.env.baklava +++ b/.env.baklava @@ -5,7 +5,6 @@ ENV_TYPE="production" GETH_VERBOSITY=2 GETH_ENABLE_METRICS=true -VM_BASED=false KUBERNETES_CLUSTER_NAME="baklavastaging" KUBERNETES_CLUSTER_ZONE="us-west1-a" diff --git a/.env.oracledev b/.env.oracledev index 6676cba2c1e..f9ddf450abf 100644 --- a/.env.oracledev +++ b/.env.oracledev @@ -61,7 +61,6 @@ CLUSTER_CREATION_FLAGS="--enable-autoscaling --min-nodes 3 --max-nodes 8 --machi # ---- VM ---- -VM_BASED=false # ---- Blockscout ---- diff --git a/.env.rc1staging b/.env.rc1staging index 9cfd22b5566..8d651268dfe 100644 --- a/.env.rc1staging +++ b/.env.rc1staging @@ -5,9 +5,6 @@ ENV_TYPE="staging" GETH_VERBOSITY=2 GETH_ENABLE_METRICS=true -# TODO: deprecated -VM_BASED=false - KUBERNETES_CLUSTER_NAME="rc1staging" KUBERNETES_CLUSTER_ZONE="us-west1-a" CLUSTER_DOMAIN_NAME="celo-testnet" diff --git a/.env.staging b/.env.staging index e9f275a5013..4dc0ef883ca 100644 --- a/.env.staging +++ b/.env.staging @@ -37,7 +37,6 @@ GETH_NODE_DOCKER_IMAGE_TAG="8a44c2cd92200bdffce595c7558e84a39ea2bc15" GETH_VERBOSITY=2 -VM_BASED=false KUBERNETES_CLUSTER_NAME=celo-networks-dev KUBERNETES_CLUSTER_ZONE="us-west1-a" diff --git a/packages/celotool/src/cmds/deploy/destroy/cluster.ts b/packages/celotool/src/cmds/deploy/destroy/cluster.ts index b0c76a7dcee..b4cb62058d4 100644 --- a/packages/celotool/src/cmds/deploy/destroy/cluster.ts +++ b/packages/celotool/src/cmds/deploy/destroy/cluster.ts @@ -1,6 +1,6 @@ import { printReleases } from 'src/cmds/deploy/list' import { deleteCluster, getNonSystemHelmReleases, switchToClusterFromEnv } from 'src/lib/cluster' -import { EnvTypes, envVar, fetchEnv } from 'src/lib/env-utils' +import { envTypes, envVar, fetchEnv } from 'src/lib/env-utils' import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' import { DestroyArgv } from '../../deploy/destroy' @@ -13,7 +13,7 @@ export const builder = {} export const handler = async (argv: DestroyArgv) => { exitIfCelotoolHelmDryRun() const envType = fetchEnv(envVar.ENV_TYPE) - if (envType !== EnvTypes.DEVELOPMENT) { + if (envType !== envTypes.DEVELOPMENT) { console.error('You can only delete dev clusters') process.exit(1) } diff --git a/packages/celotool/src/cmds/deploy/destroy/forno.ts b/packages/celotool/src/cmds/deploy/destroy/forno.ts deleted file mode 100644 index 69ceda4e343..00000000000 --- a/packages/celotool/src/cmds/deploy/destroy/forno.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { DestroyArgv } from 'src/cmds/deploy/destroy' -import { destroyForno } from 'src/lib/forno' -import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' - -export const command = 'forno' - -export const describe = 'Destroy forno for an environment' - -type FullNodeInitialArgv = DestroyArgv - -export const handler = async (argv: FullNodeInitialArgv) => { - exitIfCelotoolHelmDryRun() - await destroyForno(argv.celoEnv) -} diff --git a/packages/celotool/src/cmds/deploy/destroy/testnet.ts b/packages/celotool/src/cmds/deploy/destroy/testnet.ts index 1970025c181..7d531742a53 100644 --- a/packages/celotool/src/cmds/deploy/destroy/testnet.ts +++ b/packages/celotool/src/cmds/deploy/destroy/testnet.ts @@ -1,5 +1,4 @@ import { switchToClusterFromEnv } from 'src/lib/cluster' -import { failIfVmBased } from 'src/lib/env-utils' import { deleteFromCluster, deleteStaticIPs, exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' import { DestroyArgv } from '../../deploy/destroy' @@ -10,7 +9,6 @@ export const builder = {} export const handler = async (argv: DestroyArgv) => { exitIfCelotoolHelmDryRun() - failIfVmBased() await switchToClusterFromEnv(argv.celoEnv) diff --git a/packages/celotool/src/cmds/deploy/destroy/vm-testnet.ts b/packages/celotool/src/cmds/deploy/destroy/vm-testnet.ts deleted file mode 100644 index 0c6b1d5e9a0..00000000000 --- a/packages/celotool/src/cmds/deploy/destroy/vm-testnet.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { switchToClusterFromEnv } from 'src/lib/cluster' -import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' -import { removePrometheus } from 'src/lib/prometheus' -import { destroy } from 'src/lib/vm-testnet-utils' -import { DestroyArgv } from '../../deploy/destroy' - -export const command = 'vm-testnet' -export const describe = 'destroy an existing VM-based testnet' -export const builder = {} - -export const handler = async (argv: DestroyArgv) => { - exitIfCelotoolHelmDryRun() - await switchToClusterFromEnv(argv.celoEnv) - await destroy(argv.celoEnv) - await removePrometheus() -} diff --git a/packages/celotool/src/cmds/deploy/initial/forno.ts b/packages/celotool/src/cmds/deploy/initial/forno.ts deleted file mode 100644 index a3109b128d8..00000000000 --- a/packages/celotool/src/cmds/deploy/initial/forno.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { InitialArgv } from 'src/cmds/deploy/initial' -import { deployForno } from 'src/lib/forno' -import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' - -export const command = 'forno' - -export const describe = 'Deploy forno for an environment' - -type FullNodeInitialArgv = InitialArgv - -export const handler = async (argv: FullNodeInitialArgv) => { - exitIfCelotoolHelmDryRun() - await deployForno(argv.celoEnv) -} diff --git a/packages/celotool/src/cmds/deploy/initial/testnet.ts b/packages/celotool/src/cmds/deploy/initial/testnet.ts index 0ae62d6285a..5374986ea93 100644 --- a/packages/celotool/src/cmds/deploy/initial/testnet.ts +++ b/packages/celotool/src/cmds/deploy/initial/testnet.ts @@ -1,5 +1,4 @@ import { createClusterIfNotExists, setupCluster, switchToClusterFromEnv } from 'src/lib/cluster' -import { failIfVmBased } from 'src/lib/env-utils' import { createStaticIPs, installHelmChart, @@ -34,8 +33,6 @@ export const builder = (argv: yargs.Argv) => { } export const handler = async (argv: TestnetInitialArgv) => { - failIfVmBased() - const createdCluster = await createClusterIfNotExists() await switchToClusterFromEnv(argv.celoEnv) diff --git a/packages/celotool/src/cmds/deploy/initial/vm-testnet.ts b/packages/celotool/src/cmds/deploy/initial/vm-testnet.ts deleted file mode 100644 index 8fc3159d6a8..00000000000 --- a/packages/celotool/src/cmds/deploy/initial/vm-testnet.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { createClusterIfNotExists, setupCluster, switchToClusterFromEnv } from 'src/lib/cluster' -import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' -import { installPrometheusIfNotExists } from 'src/lib/prometheus' -import { deploy } from 'src/lib/vm-testnet-utils' -import yargs from 'yargs' -import { InitialArgv } from '../../deploy/initial' - -export const command = 'vm-testnet' -export const describe = 'upgrade a testnet on a VM' - -type VmTestnetArgv = InitialArgv & { - skipSecretGeneration: boolean - useExistingGenesis: boolean -} - -export const builder = (argv: yargs.Argv) => { - return argv - .option('skipSecretGeneration', { - describe: - 'Skips the generation of secrets. Use sparingly, this is intended to save deploy time if you are certain no secrets will have changed.', - default: false, - type: 'boolean', - }) - .option('useExistingGenesis', { - type: 'boolean', - description: 'Instead of generating a new genesis, use an existing genesis in GCS', - default: false, - }) -} - -export const handler = async (argv: VmTestnetArgv) => { - exitIfCelotoolHelmDryRun() - // deploy VM testnet with Terraform - await deploy(argv.celoEnv, !argv.skipSecretGeneration, argv.useExistingGenesis) - - // set up Kubernetes cluster that will have prometheus to stackdriver statefulset - const createdCluster = await createClusterIfNotExists() - await switchToClusterFromEnv(argv.celoEnv) - await setupCluster(argv.celoEnv, createdCluster) - await installPrometheusIfNotExists() -} diff --git a/packages/celotool/src/cmds/deploy/upgrade/forno.ts b/packages/celotool/src/cmds/deploy/upgrade/forno.ts deleted file mode 100644 index 57d3a3a6557..00000000000 --- a/packages/celotool/src/cmds/deploy/upgrade/forno.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { UpgradeArgv } from 'src/cmds/deploy/upgrade' -import { deployForno } from 'src/lib/forno' -import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' - -export const command = 'forno' - -export const describe = 'Upgrade forno for an environment' - -type FullNodeInitialArgv = UpgradeArgv - -export const handler = async (argv: FullNodeInitialArgv) => { - exitIfCelotoolHelmDryRun() - await deployForno(argv.celoEnv) -} diff --git a/packages/celotool/src/cmds/deploy/upgrade/testnet.ts b/packages/celotool/src/cmds/deploy/upgrade/testnet.ts index 60f041d3044..213d22aeaeb 100644 --- a/packages/celotool/src/cmds/deploy/upgrade/testnet.ts +++ b/packages/celotool/src/cmds/deploy/upgrade/testnet.ts @@ -1,5 +1,4 @@ import { switchToClusterFromEnv } from 'src/lib/cluster' -import { failIfVmBased } from 'src/lib/env-utils' import { isCelotoolHelmDryRun, resetAndUpgradeHelmChart, @@ -36,8 +35,6 @@ export const builder = (argv: yargs.Argv) => { } export const handler = async (argv: TestnetArgv) => { - failIfVmBased() - await switchToClusterFromEnv(argv.celoEnv) await upgradeStaticIPs(argv.celoEnv) diff --git a/packages/celotool/src/cmds/deploy/upgrade/vm-testnet.ts b/packages/celotool/src/cmds/deploy/upgrade/vm-testnet.ts deleted file mode 100644 index 79ea6309a0e..00000000000 --- a/packages/celotool/src/cmds/deploy/upgrade/vm-testnet.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { switchToClusterFromEnv } from 'src/lib/cluster' -import { exitIfCelotoolHelmDryRun } from 'src/lib/helm_deploy' -import { upgradePrometheus } from 'src/lib/prometheus' -import { deploy, taintTestnet, untaintTestnet } from 'src/lib/vm-testnet-utils' -import yargs from 'yargs' -import { UpgradeArgv } from '../../deploy/upgrade' - -export const command = 'vm-testnet' -export const describe = 'upgrade a testnet on a VM' - -type VmTestnetArgv = UpgradeArgv & { - reset: boolean - skipSecretGeneration: boolean - useExistingGenesis: boolean -} - -export const builder = (argv: yargs.Argv) => { - return argv - .option('reset', { - describe: 'recreates all nodes and deletes any chain data in persistent disks', - default: false, - type: 'boolean', - }) - .option('skipSecretGeneration', { - describe: - 'Skips the generation of secrets. Use sparingly, this is intended to save deploy time if you are certain no secrets will have changed.', - default: false, - type: 'boolean', - }) - .option('useExistingGenesis', { - type: 'boolean', - description: 'Instead of generating a new genesis, use an existing genesis in GCS', - default: false, - }) -} - -export const handler = async (argv: VmTestnetArgv) => { - exitIfCelotoolHelmDryRun() - await switchToClusterFromEnv(argv.celoEnv) - - let onDeployFailed = () => Promise.resolve() - if (argv.reset === true) { - onDeployFailed = () => untaintTestnet(argv.celoEnv) - await taintTestnet(argv.celoEnv) - } - await deploy(argv.celoEnv, !argv.skipSecretGeneration, argv.useExistingGenesis, onDeployFailed) - await upgradePrometheus() -} diff --git a/packages/celotool/src/cmds/ssh-vm-node.ts b/packages/celotool/src/cmds/ssh-vm-node.ts deleted file mode 100644 index 9592584ce74..00000000000 --- a/packages/celotool/src/cmds/ssh-vm-node.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { addCeloEnvMiddleware, CeloEnvArgv } from 'src/lib/env-utils' -import { getNodeVmName, getVmSshCommand, indexCoercer } from 'src/lib/vm-testnet-utils' -import yargs from 'yargs' - -export const command = 'ssh-vm-node [nodeIndex]' - -export const describe = - 'Generates a command to ssh into a vm-testnet node. To execute the ssh command, run `eval $()`' - -interface SshVmNodeArgv extends CeloEnvArgv { - nodeType: 'validator' | 'tx-node' | 'tx-node-private' | 'bootnode' | 'proxy' - nodeIndex?: number -} - -interface CheckArgs { - 'celo-env': unknown - nodeType: string | undefined - nodeIndex?: string -} - -export const builder = (argv: yargs.Argv) => { - const choices: readonly string[] = [ - 'validator', - 'tx-node', - 'tx-node-private', - 'bootnode', - 'proxy', - ] - return addCeloEnvMiddleware(argv) - .positional('nodeType', { - describe: 'Type of node', - choices, - }) - .positional('nodeIndex', { - describe: 'Index of the node. Only needed for validator or tx-node', - type: 'string', - coerce: indexCoercer, - }) - .check((checkArgv: CheckArgs) => { - const requiresIndex = checkArgv.nodeType !== 'bootnode' - if (requiresIndex && checkArgv.nodeIndex === undefined) { - return new Error(`nodeIndex is required for nodeType ${checkArgv.nodeType}`) - } - return true - }) -} - -export const handler = async (argv: SshVmNodeArgv) => { - const instanceName = await getNodeVmName(argv.celoEnv, argv.nodeType, argv.nodeIndex) - console.info(getVmSshCommand(instanceName)) -} diff --git a/packages/celotool/src/cmds/vm-exec.ts b/packages/celotool/src/cmds/vm-exec.ts deleted file mode 100644 index 160f1f75b68..00000000000 --- a/packages/celotool/src/cmds/vm-exec.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { execCmd } from 'src/lib/cmd-utils' -import { - addCeloEnvMiddleware, - CeloEnvArgv, - envVar, - failIfNotVmBased, - fetchEnv, -} from 'src/lib/env-utils' -import { getProxiesPerValidator } from 'src/lib/testnet-utils' -import { getNodeVmName, getVmSshCommand, indexCoercer, ProxyIndex } from 'src/lib/vm-testnet-utils' -import yargs from 'yargs' - -export const command = 'vm-exec' - -export const describe = 'SSH and exec commands on all or individual nodes in a VM-based env' - -interface ValidatorsExecArgv extends CeloEnvArgv { - nodeType: string - docker: string - cmd: string - only: number | ProxyIndex - from: number | ProxyIndex - to: number | ProxyIndex -} - -export const builder = (argv: yargs.Argv) => { - return addCeloEnvMiddleware(argv) - .option('nodeType', { - describe: 'Type of node', - choices: ['validator', 'tx-node', 'tx-node-private', 'bootnode', 'proxy'], - type: 'string', - }) - .option('docker', { - type: 'string', - description: 'Operation to run on the docker container {start|stop|restart}', - default: 'restart', - }) - .option('cmd', { - type: 'string', - description: 'Arbitrary command to exec', - default: null, - }) - .option('only', { - type: 'string', - description: - 'Index of the only node to exec on. If the node is a proxy, the validator and proxy indices must both be specified as `:`', - default: null, - coerce: indexCoercer, - }) - .option('from', { - type: 'string', - description: - 'Index of the node to start on when exec-ing over a range. If the node is a proxy, the validator and proxy indices must both be specified as `:`', - default: '0', - coerce: indexCoercer, - }) - .option('to', { - type: 'string', - description: - 'Index of the node to end on when exec-ing over a range (not inclusive). If the node is a proxy, the validator and proxy indices must both be specified as `:`. Defaults to the max index for the nodeType.', - default: '-1', - coerce: indexCoercer, - }) -} - -export const handler = async (argv: ValidatorsExecArgv) => { - failIfNotVmBased() - - const project = fetchEnv(envVar.TESTNET_PROJECT_NAME) - const zone = fetchEnv(envVar.KUBERNETES_CLUSTER_ZONE) - - const cmd = argv.cmd === null ? `sudo docker ${argv.docker} geth` : argv.cmd - - console.info( - `Running on validators.\n` + - `Cmd: ${cmd}\n` + - `Env: ${argv.celoEnv}\n` + - `Project: ${project}\n` + - `Zone: ${zone}\n` + - `Node Type: ${argv.nodeType}` - ) - - // For proxy / tx-nodes that have random suffixes, we are forced to run a - // gcloud command and await it in order to get the full instance name. - // Because of this, we end up calling the SSH command, and then moving on to get the - // next instance name, which takes time, so the previous SSH command is nearly finished. - // By doing this in two steps, we more closely make the exec across all instances - // happen in parallel - const instanceNames: string[] = [] - if (argv.only === null) { - let to: number | ProxyIndex = argv.to - - if (typeof to === 'number' && to < 0) { - to = getMaxNodeIndex(argv.nodeType) - } - - console.info('Max Node Index:', getMaxNodeIndex(argv.nodeType)) - console.info('From Index:', argv.from) - console.info('To Index:', to) - - const indexIterator = createIndexIterator(argv.from, to) - let index = indexIterator.next() - while (!index.done) { - const instanceName = await getNodeVmName(argv.celoEnv, argv.nodeType, index.value) - instanceNames.push(instanceName) - index = indexIterator.next() - } - } else { - console.info(`Only Index: ${argv.only}`) - const instanceName = await getNodeVmName(argv.celoEnv, argv.nodeType, argv.only) - instanceNames.push(instanceName) - } - - const runCmds = [] - for (const instanceName of instanceNames) { - runCmds.push(runSshCommand(instanceName, cmd)) - } - - await Promise.all(runCmds) - - console.info('Done.') -} - -async function runSshCommand(instanceName: string, cmd: string) { - const bareSshCmd = getVmSshCommand(instanceName) - const fullCmd = `${bareSshCmd} --command "${cmd}"` - console.info(`Running ${fullCmd}`) - return execCmd(fullCmd, {}, false, true) -} - -function getMaxNodeIndex(nodeType: string): number | ProxyIndex { - switch (nodeType) { - case 'validator': - return parseInt(fetchEnv(envVar.VALIDATORS), 10) - case 'tx-node': - return parseInt(fetchEnv(envVar.TX_NODES), 10) - case 'tx-node-private': - return parseInt(fetchEnv(envVar.PRIVATE_TX_NODES), 10) - case 'bootnode': - return 1 - case 'proxy': - const proxiesPerValidator = getProxiesPerValidator() - if (!proxiesPerValidator.length) { - return { - validatorIndex: 0, - proxyIndex: 0, - } - } - return { - validatorIndex: proxiesPerValidator.length - 1, - proxyIndex: proxiesPerValidator[proxiesPerValidator.length - 1], - } - default: - throw new Error('Invalid node type') - } -} - -function* createIndexIterator(from: number | ProxyIndex, to: number | ProxyIndex) { - if (typeof from !== typeof to) { - throw Error('From and to indices should be of the same type') - } - if (typeof from === 'number') { - // iterate through numeric indices - for (let i = from; i < to; i++) { - yield i - } - } else { - const proxyFrom = from as ProxyIndex - const proxyTo = to as ProxyIndex - // iterate through proxy indices - const proxiesPerValidator = getProxiesPerValidator() - const minValidatorIndex = Math.min(proxiesPerValidator.length, proxyTo.validatorIndex) - for (let valIndex = proxyFrom.validatorIndex; valIndex <= minValidatorIndex; valIndex++) { - const maxProxyIndex = - valIndex === proxyTo.validatorIndex ? proxyTo.proxyIndex : proxiesPerValidator[valIndex] - for (let proxyIndex = from.proxyIndex; proxyIndex < maxProxyIndex; proxyIndex++) { - const index: ProxyIndex = { - validatorIndex: valIndex, - proxyIndex, - } - yield index - } - } - } -} diff --git a/packages/celotool/src/lib/cluster.ts b/packages/celotool/src/lib/cluster.ts index 7baed1ad446..95f6661cb75 100644 --- a/packages/celotool/src/lib/cluster.ts +++ b/packages/celotool/src/lib/cluster.ts @@ -1,7 +1,7 @@ import sleep from 'sleep-promise' import { execCmd, execCmdWithExitOnFailure } from './cmd-utils' import { getClusterConfigForContext, switchToContextCluster } from './context-utils' -import { doCheckOrPromptIfStagingOrProduction, EnvTypes, envVar, fetchEnv } from './env-utils' +import { doCheckOrPromptIfStagingOrProduction, envTypes, envVar, fetchEnv } from './env-utils' import { checkHelmVersion, createAndUploadBackupSecretIfNotExists, @@ -11,10 +11,10 @@ import { installCertManagerAndNginx, installGCPSSDStorageClass, isCelotoolHelmDryRun, + networkName, } from './helm_deploy' import { createServiceAccountIfNotExists } from './service-account-utils' import { outputIncludes, switchToProjectFromEnv } from './utils' -import { networkName } from './vm-testnet-utils' const SYSTEM_HELM_RELEASES = [ 'nginx-ingress-release', @@ -137,7 +137,7 @@ export async function setupCluster(celoEnv: string, createdCluster: boolean) { await installCertManagerAndNginx(celoEnv) - if (envType !== EnvTypes.DEVELOPMENT) { + if (envType !== envTypes.DEVELOPMENT) { console.info('Installing metric tools installation') await installAndEnableMetricsDeps(true) } else { @@ -199,7 +199,7 @@ export async function setClusterLabels(celoEnv: string) { ) } await labelfn('environment', envType) - await labelfn('envtype', envType === EnvTypes.PRODUCTION ? 'production' : 'nonproduction') + await labelfn('envtype', envType === envTypes.PRODUCTION ? 'production' : 'nonproduction') await labelfn('envinstance', celoEnv) } diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts index d57f4c19f0a..e70ebcaf44a 100644 --- a/packages/celotool/src/lib/env-utils.ts +++ b/packages/celotool/src/lib/env-utils.ts @@ -143,7 +143,6 @@ export enum envVar { VALIDATOR_ZERO_GENESIS_BALANCE = 'VALIDATOR_ZERO_GENESIS_BALANCE', VALIDATORS = 'VALIDATORS', VALIDATORS_ROLLING_UPDATE_PARTITION = 'VALIDATORS_ROLLING_UPDATE_PARTITION', - VM_BASED = 'VM_BASED', VOTING_BOT_BALANCE = 'VOTING_BOT_BALANCE', VOTING_BOT_CHANGE_BASELINE = 'VOTING_BOT_CHANGE_BASELINE', VOTING_BOT_CRON_SCHEDULE = 'VOTING_BOT_CRON_SCHEDULE', @@ -227,7 +226,7 @@ export enum DynamicEnvVar { PROM_REMOTE_WRITE_URL = '{{ context }}_PROM_REMOTE_WRITE_URL', } -export enum EnvTypes { +export enum envTypes { DEVELOPMENT = 'development', INTEGRATION = 'integration', STAGING = 'staging', @@ -291,7 +290,7 @@ export function validateAndSwitchToEnv(celoEnv: string) { } export function isProduction() { - return fetchEnv(envVar.ENV_TYPE).toLowerCase() === EnvTypes.PRODUCTION + return fetchEnv(envVar.ENV_TYPE).toLowerCase() === envTypes.PRODUCTION } export function isValidCeloEnv(celoEnv: string) { @@ -375,21 +374,3 @@ export function addCeloEnvMiddleware(argv: yargs.Argv) { .middleware([celoEnvMiddleware]) ) } - -export function isVmBased() { - return fetchEnv(envVar.VM_BASED) === 'true' -} - -export function failIfNotVmBased() { - if (!isVmBased()) { - console.error('The celo env is not intended for a VM-based testnet, aborting') - process.exit(1) - } -} - -export function failIfVmBased() { - if (isVmBased()) { - console.error('The celo env is intended for a VM-based testnet, aborting') - process.exit(1) - } -} diff --git a/packages/celotool/src/lib/forno.ts b/packages/celotool/src/lib/forno.ts deleted file mode 100644 index ef97fc7fc8e..00000000000 --- a/packages/celotool/src/lib/forno.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { execCmd } from './cmd-utils' -import { coerceContext, getClusterManagerForContext, readableContext } from './context-utils' -import { envVar, fetchEnv } from './env-utils' -import { CloudProvider } from './k8s-cluster/base' -import { GCPClusterConfig } from './k8s-cluster/gcp' -import { TerraformVars } from './terraform' -import { deployModule, destroyModule } from './vm-testnet-utils' - -const FORNO_TERRAFORM_MODULE_NAME = 'forno' - -export async function deployForno(celoEnv: string) { - const contexts: string[] = fetchEnv(envVar.FORNO_FULL_NODE_CONTEXTS).split(',').map(coerceContext) - console.info('Deploying Forno with full node contexts:', contexts) - const terraformVars: TerraformVars = await getFornoTerraformVars(celoEnv, contexts) - // This prints the global IP address for forno - await deployModule(celoEnv, FORNO_TERRAFORM_MODULE_NAME, terraformVars) - console.info( - 'Note: in order to have an SSL certificate be properly provisioned, DNS entries for the relevant domains must point to the printed IP above.' - ) -} - -export async function destroyForno(celoEnv: string) { - const contexts: string[] = fetchEnv(envVar.FORNO_FULL_NODE_CONTEXTS).split(',').map(coerceContext) - console.info('DESTROYING Forno') - const terraformVars: TerraformVars = await getFornoTerraformVars(celoEnv, contexts) - await destroyModule(celoEnv, FORNO_TERRAFORM_MODULE_NAME, terraformVars) -} - -interface ContextInfoTerraformVars { - service_network_endpoint_group_name: string - zone: string -} - -async function getFornoTerraformVars(celoEnv: string, contexts: string[]): Promise { - let gcloudProject: string | undefined - const getContextInfos = async ( - port: number, - service: string, - namespace: string - ): Promise<{ [context: string]: ContextInfoTerraformVars }> => - contexts.reduce(async (aggPromise, context: string) => { - const agg = await aggPromise - const clusterManager = getClusterManagerForContext(celoEnv, context) - if (clusterManager.cloudProvider !== CloudProvider.GCP) { - throw Error( - `Forno only accepts GCP contexts, context ${context} is ${clusterManager.cloudProvider}` - ) - } - const contextGcloudProject = (clusterManager.clusterConfig as GCPClusterConfig).projectName - // Require all the contexts to have the same project - if (gcloudProject === undefined) { - gcloudProject = contextGcloudProject - } else if (gcloudProject !== contextGcloudProject) { - throw Error(`All contexts must be in the same Google Cloud project`) - } - // Rather than using clusterManager.kubernetesContextName we switch to the - // cluster to account for the case where this user has not gotten the - // context for the cluster yet. - await clusterManager.switchToClusterContext(true) - const [output] = await execCmd( - `kubectl get svc ${service} -n ${namespace} -o jsonpath="{.metadata.annotations.cloud\\.google\\.com/neg-status}"` - ) - if (!output.trim()) { - throw Error(`Expected cloud.google.com/neg-status annotation for service ${service}`) - } - const outputParsed = JSON.parse(output) - if (!outputParsed.network_endpoint_groups[port] || !outputParsed.zones.length) { - throw Error( - `Expected NEG for ${port} and > 0 zones, instead got NEG: ${outputParsed.network_endpoint_groups[port]} and zones ${outputParsed.zones}` - ) - } - return { - ...agg, - [readableContext(context)]: { - service_network_endpoint_group_name: outputParsed.network_endpoint_groups[port], - // Only expect a single zone - zone: outputParsed.zones[0], - }, - } - }, Promise.resolve({})) - - // Make sure each domain ends with a period - const domains = fetchEnv(envVar.FORNO_DOMAINS) - .split(',') - .map((domain: string) => { - if (!domain.endsWith('.')) { - return `${domain}.` - } - return domain - }) - const HTTP_RPC_PORT = 8545 - const WS_RPC_PORT = 8546 - const KONG_RPC_PORT = 80 - const contextInfosHttp = await getContextInfos(HTTP_RPC_PORT, `${celoEnv}-fullnodes-rpc`, celoEnv) - const contextInfosWs = await getContextInfos(WS_RPC_PORT, `${celoEnv}-fullnodes-rpc`, celoEnv) - const contextInfosKong = await getContextInfos(KONG_RPC_PORT, 'kong', 'kong') - const bannedCIDRs = fetchEnv(envVar.FORNO_BANNED_CIDR).split(',') - - return { - backend_max_requests_per_second: '100', - backend_max_requests_per_second_kong: '10000', - celo_env: celoEnv, - context_info_http: JSON.stringify(contextInfosHttp), - context_info_ws: JSON.stringify(contextInfosWs), - context_info_kong: JSON.stringify(contextInfosKong), - gcloud_credentials_path: fetchEnv(envVar.GOOGLE_APPLICATION_CREDENTIALS), - gcloud_project: gcloudProject!, - ssl_cert_domains: JSON.stringify(domains), - banned_cidr: JSON.stringify(bannedCIDRs), - vpc_network_name: fetchEnv(envVar.FORNO_VPC_NETWORK_NAME), - } -} diff --git a/packages/celotool/src/lib/geth.ts b/packages/celotool/src/lib/geth.ts index c6e4ca80866..db0280ee571 100644 --- a/packages/celotool/src/lib/geth.ts +++ b/packages/celotool/src/lib/geth.ts @@ -16,7 +16,7 @@ import Web3 from 'web3' import { Admin } from 'web3-eth-admin' import { spawnCmd, spawnCmdWithExitOnFailure } from './cmd-utils' import { convertToContractDecimals } from './contract-utils' -import { envVar, fetchEnv, isVmBased } from './env-utils' +import { envVar, fetchEnv } from './env-utils' import { AccountType, generateGenesis, @@ -29,7 +29,6 @@ import { retrieveClusterIPAddress, retrieveIPAddress } from './helm_deploy' import { GethInstanceConfig } from './interfaces/geth-instance-config' import { GethRunConfig } from './interfaces/geth-run-config' import { ensure0x } from './utils' -import { getTestnetOutputs } from './vm-testnet-utils' export async function unlockAccount( web3: Web3, @@ -91,29 +90,19 @@ export const getBootnodeEnode = async (namespace: string) => { } export const retrieveBootnodeIPAddress = async (namespace: string) => { - if (isVmBased()) { - const outputs = await getTestnetOutputs(namespace) - return outputs.bootnode_ip_address.value + // Baklava bootnode address comes from VM and has an different name (not possible to update name after creation) + const resourceName = + namespace === 'baklava' ? `${namespace}-bootnode-address` : `${namespace}-bootnode` + if (fetchEnv(envVar.STATIC_IPS_FOR_GETH_NODES) === 'true') { + return retrieveIPAddress(resourceName) } else { - // Baklava bootnode address comes from VM and has an different name (not possible to update name after creation) - const resourceName = - namespace === 'baklava' ? `${namespace}-bootnode-address` : `${namespace}-bootnode` - if (fetchEnv(envVar.STATIC_IPS_FOR_GETH_NODES) === 'true') { - return retrieveIPAddress(resourceName) - } else { - return retrieveClusterIPAddress('service', resourceName, namespace) - } + return retrieveClusterIPAddress('service', resourceName, namespace) } } const retrieveTxNodeAddresses = async (namespace: string, txNodesNum: number) => { - if (isVmBased()) { - const outputs = await getTestnetOutputs(namespace) - return outputs.tx_node_ip_addresses.value - } else { - const txNodesRange = range(0, txNodesNum) - return Promise.all(txNodesRange.map((i) => retrieveIPAddress(`${namespace}-tx-nodes-${i}`))) - } + const txNodesRange = range(0, txNodesNum) + return Promise.all(txNodesRange.map((i) => retrieveIPAddress(`${namespace}-tx-nodes-${i}`))) } const getEnodesWithIpAddresses = async (namespace: string, getExternalIP: boolean) => { diff --git a/packages/celotool/src/lib/helm_deploy.ts b/packages/celotool/src/lib/helm_deploy.ts index ed32bb1fd90..ac5975c0964 100644 --- a/packages/celotool/src/lib/helm_deploy.ts +++ b/packages/celotool/src/lib/helm_deploy.ts @@ -15,7 +15,7 @@ import { spawnCmd, spawnCmdWithExitOnFailure, } from './cmd-utils' -import { EnvTypes, envVar, fetchEnv, fetchEnvOrFallback, monorepoRoot } from './env-utils' +import { envTypes, envVar, fetchEnv, fetchEnvOrFallback, monorepoRoot } from './env-utils' import { ensureAuthenticatedGcloudAccount } from './gcloud_utils' import { generateGenesisFromEnv } from './generate_utils' import { @@ -110,7 +110,7 @@ export async function createCloudSQLInstance(celoEnv: string, instanceName: stri } const envType = fetchEnv(envVar.ENV_TYPE) - if (envType !== EnvTypes.DEVELOPMENT) { + if (envType !== envTypes.DEVELOPMENT) { try { await execCmdWithExitOnFailure( `gcloud sql instances create ${instanceName}-replica --master-instance-name=${instanceName} --zone ${fetchEnv( @@ -1398,3 +1398,11 @@ async function generateMyCeloGenesis(): Promise { await spawnCmd('rm', ['-rf', celoBlockchainDir], { silent: true }) return genesisContent } + +function useDefaultNetwork() { + return fetchEnv(envVar.KUBERNETES_CLUSTER_NAME) === 'celo-networks-dev' +} + +export function networkName(celoEnv: string) { + return useDefaultNetwork() ? 'default' : `${celoEnv}-network` +} diff --git a/packages/celotool/src/lib/mock-oracle.ts b/packages/celotool/src/lib/mock-oracle.ts index 83fefa45736..778bb206cf6 100644 --- a/packages/celotool/src/lib/mock-oracle.ts +++ b/packages/celotool/src/lib/mock-oracle.ts @@ -1,7 +1,6 @@ -import { envVar, fetchEnv, isVmBased } from 'src/lib/env-utils' +import { envVar, fetchEnv } from 'src/lib/env-utils' import { getPrivateTxNodeClusterIP } from 'src/lib/geth' import { installGenericHelmChart, removeGenericHelmChart } from 'src/lib/helm_deploy' -import { getInternalTxNodeLoadBalancerIP } from 'src/lib/vm-testnet-utils' const helmChartPath = '../helm-charts/mock-oracle' @@ -18,9 +17,7 @@ export async function removeHelmRelease(celoEnv: string) { } async function helmParameters(celoEnv: string) { - const nodeIp = isVmBased() - ? await getInternalTxNodeLoadBalancerIP(celoEnv) - : await getPrivateTxNodeClusterIP(celoEnv) + const nodeIp = await getPrivateTxNodeClusterIP(celoEnv) const nodeUrl = `http://${nodeIp}:8545` return [ `--set celotool.image.repository=${fetchEnv(envVar.CELOTOOL_DOCKER_IMAGE_REPOSITORY)}`, diff --git a/packages/celotool/src/lib/port_forward.ts b/packages/celotool/src/lib/port_forward.ts index a0aac538557..297a326fab0 100644 --- a/packages/celotool/src/lib/port_forward.ts +++ b/packages/celotool/src/lib/port_forward.ts @@ -1,7 +1,6 @@ /* tslint:disable: no-console */ import { ChildProcess, spawnSync } from 'child_process' import { execBackgroundCmd, execCmd } from './cmd-utils' -import { envVar, fetchEnv, isVmBased } from './env-utils' function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)) @@ -13,19 +12,7 @@ const PORT_CONTROL_CMD = 'nc -z 127.0.0.1 8545' const DEFAULT_COMPONENT = 'validators' async function getPortForwardCmd(celoEnv: string, component?: string, ports = defaultPortsString) { - if (isVmBased()) { - return Promise.resolve(getVmPortForwardCmd(celoEnv, component, ports)) - } else { - return getKubernetesPortForwardCmd(celoEnv, component, ports) - } -} - -function getVmPortForwardCmd(celoEnv: string, machine = 'validator-0', ports = defaultPortsString) { - const zone = fetchEnv(envVar.KUBERNETES_CLUSTER_ZONE) - // this command expects port mappings to be of the form `[localPort]:localhost:[remotePort]` - const portMappings = ports.replace(/:/g, ':localhost:').split(' ') - const portsWithFlags = portMappings.map((mapping) => `-L ${mapping}`).join(' ') - return `gcloud compute ssh --zone ${zone} ${celoEnv}-${machine} -- -N ${portsWithFlags}` + return getKubernetesPortForwardCmd(celoEnv, component, ports) } async function getKubernetesPortForwardCmd( diff --git a/packages/celotool/src/lib/transaction-metrics-exporter.ts b/packages/celotool/src/lib/transaction-metrics-exporter.ts index 7ca1356011b..4b79a2325eb 100644 --- a/packages/celotool/src/lib/transaction-metrics-exporter.ts +++ b/packages/celotool/src/lib/transaction-metrics-exporter.ts @@ -1,10 +1,9 @@ -import { envVar, fetchEnv, fetchEnvOrFallback, isVmBased } from 'src/lib/env-utils' +import { envVar, fetchEnv, fetchEnvOrFallback } from 'src/lib/env-utils' import { installGenericHelmChart, removeGenericHelmChart, upgradeGenericHelmChart, } from 'src/lib/helm_deploy' -import { getInternalTxNodeLoadBalancerIP } from 'src/lib/vm-testnet-utils' const chartDir = '../helm-charts/transaction-metrics-exporter/' @@ -58,8 +57,5 @@ async function helmParameters(celoEnv: string) { '' )}`, ] - if (isVmBased()) { - params.push(`--set web3Provider="ws://${await getInternalTxNodeLoadBalancerIP(celoEnv)}:8546"`) - } return params } diff --git a/packages/celotool/src/lib/utils.ts b/packages/celotool/src/lib/utils.ts index 3db31d6a569..40e4bf74beb 100644 --- a/packages/celotool/src/lib/utils.ts +++ b/packages/celotool/src/lib/utils.ts @@ -2,9 +2,8 @@ import sleep from 'sleep-promise' import yargs from 'yargs' import { switchToClusterFromEnv } from './cluster' import { execCmdWithExitOnFailure } from './cmd-utils' -import { envVar, fetchEnv, isVmBased } from './env-utils' +import { envVar, fetchEnv } from './env-utils' import { retrieveIPAddress } from './helm_deploy' -import { getTestnetOutputs } from './vm-testnet-utils' export async function outputIncludes(cmd: string, matchString: string, matchMessage?: string) { const [stdout] = await execCmdWithExitOnFailure(cmd) @@ -18,12 +17,7 @@ export async function outputIncludes(cmd: string, matchString: string, matchMess } export async function retrieveTxNodeIpAddress(celoEnv: string, txNodeIndex: number) { - if (isVmBased()) { - const outputs = await getTestnetOutputs(celoEnv) - return outputs.tx_node_ip_addresses.value[txNodeIndex] - } else { - return retrieveIPAddress(`${celoEnv}-tx-nodes-${txNodeIndex}`) - } + return retrieveIPAddress(`${celoEnv}-tx-nodes-${txNodeIndex}`) } export async function getVerificationPoolConfig(celoEnv: string) { diff --git a/packages/celotool/src/lib/vm-testnet-utils.ts b/packages/celotool/src/lib/vm-testnet-utils.ts deleted file mode 100644 index 935181b784c..00000000000 --- a/packages/celotool/src/lib/vm-testnet-utils.ts +++ /dev/null @@ -1,559 +0,0 @@ -import sleep from 'sleep-promise' -import { execCmd } from './cmd-utils' -import { confirmAction, envVar, fetchEnv, fetchEnvOrFallback } from './env-utils' -import { - AccountType, - generateGenesisFromEnv, - generatePrivateKey, - generatePublicKey, - getAddressFromEnv, - privateKeyToAddress, - privateKeyToPublicKey, -} from './generate_utils' -import { - applyTerraformModule, - destroyTerraformModule, - getTerraformModuleOutputs, - initTerraformModule, - planTerraformModule, - showTerraformModulePlan, - taintTerraformModuleResource, - TerraformVars, - untaintTerraformModuleResource, -} from './terraform' -import { - getGenesisBlockFromGoogleStorage, - getProxiesPerValidator, - getProxyName, - uploadDataToGoogleStorage, - uploadGenesisBlockToGoogleStorage, - uploadTestnetInfoToGoogleStorage, -} from './testnet-utils' - -export interface ProxyIndex { - validatorIndex: number - proxyIndex: number -} - -// Keys = gcloud project name -const projectConfig = { - 'celo-testnet': { - secretsBucketName: 'celo-testnet-secrets', - stateBucketName: 'celo_tf_state', - }, - 'celo-testnet-production': { - secretsBucketName: 'celo-testnet-secrets-prod', - stateBucketName: 'celo_tf_state_prod', - }, -} - -const testnetTerraformModule = 'testnet' -const testnetNetworkTerraformModule = 'testnet-network' - -interface NodeSecrets { - ACCOUNT_ADDRESS: string - BOOTNODE_ENODE_ADDRESS: string - PRIVATE_KEY: string - PROXIED_VALIDATOR_ADDRESS?: string - PROXY_ENODE_ADDRESSES?: string - [envVar.GETH_ACCOUNT_SECRET]: string - [envVar.MNEMONIC]: string -} - -// The keys correspond to the variable names that Terraform expects and -// the values correspond to the names of the appropriate env variables -const testnetEnvVars: TerraformVars = { - block_time: envVar.BLOCK_TIME, - celo_env: envVar.CELOTOOL_CELOENV, - gcloud_credentials_path: envVar.GOOGLE_APPLICATION_CREDENTIALS, - gcloud_project: envVar.TESTNET_PROJECT_NAME, - geth_verbosity: envVar.GETH_VERBOSITY, - geth_bootnode_docker_image_repository: envVar.GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY, - geth_bootnode_docker_image_tag: envVar.GETH_BOOTNODE_DOCKER_IMAGE_TAG, - geth_metrics: envVar.GETH_ENABLE_METRICS, - geth_node_docker_image_repository: envVar.GETH_NODE_DOCKER_IMAGE_REPOSITORY, - geth_node_docker_image_tag: envVar.GETH_NODE_DOCKER_IMAGE_TAG, - in_memory_discovery_table: envVar.IN_MEMORY_DISCOVERY_TABLE, - istanbul_request_timeout_ms: envVar.ISTANBUL_REQUEST_TIMEOUT_MS, - network_id: envVar.NETWORK_ID, - private_tx_node_count: envVar.PRIVATE_TX_NODES, - node_disk_size_gb: envVar.NODE_DISK_SIZE_GB, - private_node_disk_size_gb: envVar.PRIVATE_NODE_DISK_SIZE_GB, - tx_node_count: envVar.TX_NODES, - validator_count: envVar.VALIDATORS, -} - -const testnetNetworkEnvVars: TerraformVars = { - celo_env: envVar.CELOTOOL_CELOENV, - gcloud_credentials_path: envVar.GOOGLE_APPLICATION_CREDENTIALS, - gcloud_project: envVar.TESTNET_PROJECT_NAME, -} - -// Resources that are tainted when upgrade-resetting -const testnetResourcesToReset = [ - // bootnode - 'module.bootnode.google_compute_instance.bootnode', - // validators - 'module.validator.google_compute_instance.validator.*', - 'module.validator.google_compute_disk.validator.*', - // validator proxies - 'module.validator.module.proxy.random_id.full_node.*', - 'module.validator.module.proxy.google_compute_instance.full_node.*', - 'module.validator.module.proxy.random_id.full_node_disk.*', - 'module.validator.module.proxy.google_compute_disk.full_node.*', - // tx-nodes - 'module.tx_node.random_id.full_node.*', - 'module.tx_node.google_compute_instance.full_node.*', - 'module.tx_node.random_id.full_node_disk.*', - 'module.tx_node.google_compute_disk.full_node.*', - // private tx-nodes - 'module.tx_node_private.random_id.full_node.*', - 'module.tx_node_private.google_compute_instance.full_node.*', - 'module.tx_node_private.random_id.full_node_disk.*', - 'module.tx_node_private.google_compute_disk.full_node.*', - // tx-node load balancer instance group - 'module.tx_node_lb.random_id.external', - 'module.tx_node_lb.google_compute_instance_group.external', - 'module.tx_node_lb.random_id.internal', - 'module.tx_node_lb.google_compute_instance_group.internal', -] - -export async function deploy( - celoEnv: string, - generateSecrets: boolean, - useExistingGenesis: boolean, - onConfirmFailed?: () => Promise -) { - // If we are not using the default network, we want to create/upgrade our network - if (!useDefaultNetwork()) { - console.info('First deploying the testnet VPC network') - - const networkVars: TerraformVars = getTestnetNetworkVars(celoEnv) - await deployModule(celoEnv, testnetNetworkTerraformModule, networkVars, onConfirmFailed) - } - - const testnetVars: TerraformVars = await getTestnetVars(celoEnv, useExistingGenesis) - await deployModule(celoEnv, testnetTerraformModule, testnetVars, onConfirmFailed, async () => { - if (generateSecrets) { - console.info('Generating and uploading secrets env files to Google Storage...') - await generateAndUploadSecrets(celoEnv) - } - }) - await uploadTestnetInfoToGoogleStorage(celoEnv) -} - -export async function deployModule( - celoEnv: string, - terraformModule: string, - vars: TerraformVars, - onConfirmFailed?: () => Promise, - onConfirmSuccess?: () => Promise -) { - const backendConfigVars: TerraformVars = getTerraformBackendConfigVars(celoEnv, terraformModule) - - const envType = fetchEnv(envVar.ENV_TYPE) - console.info(` - Deploying: - Terraform Module: ${terraformModule} - Celo Env: ${celoEnv} - Environment: ${envType} - `) - - console.info('Initializing...') - await initTerraformModule(terraformModule, vars, backendConfigVars) - - console.info('Planning...') - await planTerraformModule(terraformModule, vars) - - // await showTerraformModulePlan(terraformModule) - - await confirmAction( - `Are you sure you want to perform the above plan for Celo env ${celoEnv} in environment ${envType}?`, - onConfirmFailed, - onConfirmSuccess - ) - - console.info('Applying...') - await applyTerraformModule(terraformModule) -} - -export async function destroy(celoEnv: string) { - const testnetVars: TerraformVars = await getTestnetVars(celoEnv, true) - - await destroyModule(celoEnv, testnetTerraformModule, testnetVars) - - // If we are not using the default network, we want to destroy our network - if (!useDefaultNetwork()) { - console.info('Destroying the testnet VPC network') - - const networkVars: TerraformVars = getTestnetNetworkVars(celoEnv) - await destroyModule(celoEnv, testnetNetworkTerraformModule, networkVars) - } -} - -export async function destroyModule( - celoEnv: string, - terraformModule: string, - vars: TerraformVars = {} -) { - const backendConfigVars: TerraformVars = getTerraformBackendConfigVars(celoEnv, terraformModule) - - const envType = fetchEnv(envVar.ENV_TYPE) - console.info(` - Destroying: - Terraform Module: ${terraformModule} - Celo Env: ${celoEnv} - Environment: ${envType} - `) - - console.info('Initializing...') - await initTerraformModule(terraformModule, vars, backendConfigVars) - - console.info('Planning...') - await planTerraformModule(terraformModule, vars, true) - - await showTerraformModulePlan(terraformModule) - - await confirmAction(`Are you sure you want to destroy ${celoEnv} in environment ${envType}?`) - - await destroyTerraformModule(terraformModule, vars) -} - -// force the recreation of various resources upon the next deployment -export async function taintTestnet(celoEnv: string) { - console.info('Tainting testnet...') - const vars: TerraformVars = await getTestnetVars(celoEnv, true) - const backendConfigVars: TerraformVars = getTerraformBackendConfigVars( - celoEnv, - testnetTerraformModule - ) - await initTerraformModule(testnetTerraformModule, vars, backendConfigVars) - - for (const resource of testnetResourcesToReset) { - console.info(`Tainting ${resource}`) - await taintTerraformModuleResource(testnetTerraformModule, resource) - // To avoid getting errors for too many gcloud storage API requests - await sleep(2000) - } -} - -export async function untaintTestnet(celoEnv: string) { - console.info('Untainting testnet...') - const vars: TerraformVars = await getTestnetVars(celoEnv, true) - const backendConfigVars: TerraformVars = getTerraformBackendConfigVars( - celoEnv, - testnetTerraformModule - ) - await initTerraformModule(testnetTerraformModule, vars, backendConfigVars) - - for (const resource of testnetResourcesToReset) { - console.info(`Untainting ${resource}`) - await untaintTerraformModuleResource(testnetTerraformModule, resource) - // To avoid getting errors for too many gcloud storage API requests - await sleep(2000) - } -} - -export async function getTestnetOutputs(celoEnv: string) { - const vars: TerraformVars = await getTestnetVars(celoEnv, true) - const backendConfigVars: TerraformVars = getTerraformBackendConfigVars( - celoEnv, - testnetTerraformModule - ) - await initTerraformModule(testnetTerraformModule, vars, backendConfigVars) - return getTerraformModuleOutputs(testnetTerraformModule, vars) -} - -export async function getInternalTxNodeLoadBalancerIP(celoEnv: string) { - const fullCmd = getInternalTxNodeLoadBalancerIpCommand(celoEnv) - const [output] = await execCmd(fullCmd) - return output.trim() -} - -export async function getInternalValidatorIPs(celoEnv: string) { - const outputs = await getTestnetOutputs(celoEnv) - return outputs.validator_internal_ip_addresses.value -} - -export async function getInternalProxyIPs(celoEnv: string) { - const outputs = await getTestnetOutputs(celoEnv) - return outputs.proxy_internal_ip_addresses.value -} - -export async function getInternalTxNodeIPs(celoEnv: string) { - const outputs = await getTestnetOutputs(celoEnv) - return outputs.tx_node_internal_ip_addresses.value -} - -export function getTerraformBackendConfigVars(celoEnv: string, terraformModule: string) { - return { - bucket: stateBucketName(), - prefix: `${celoEnv}/${terraformModule}`, - } -} - -async function getTestnetVars(celoEnv: string, useExistingGenesis: boolean) { - let genesisContent: string = '' - if (useExistingGenesis) { - genesisContent = await getGenesisBlockFromGoogleStorage(celoEnv) - } else { - generateGenesisFromEnv() - await uploadGenesisBlockToGoogleStorage(genesisContent, celoEnv) - } - - const genesisBuffer = Buffer.from(genesisContent) - const domainName = fetchEnv(envVar.CLUSTER_DOMAIN_NAME) - return { - ...getEnvVarValues(testnetEnvVars), - // Cloud DNS for our domains only lives in celo-testnet - dns_gcloud_project: 'celo-testnet', - dns_zone_name: dnsZoneName(domainName), - ethstats_host: `${celoEnv}-ethstats.${domainName}.org`, - forno_host: `${celoEnv}-forno.${domainName}.org`, - gcloud_secrets_bucket: secretsBucketName(), - gcloud_secrets_base_path: secretsBasePath(celoEnv), - // only able to view objects for accessing secrets & modify ssl certs for forno setup - gcloud_vm_service_account_email: `terraform-testnet@${fetchEnv( - envVar.TESTNET_PROJECT_NAME - )}.iam.gserviceaccount.com`, - genesis_content_base64: genesisBuffer.toString('base64'), - // forno is the name for our setup that has tx-nodes reachable via a domain name - letsencrypt_email: 'n@celo.org', - network_name: networkName(celoEnv), - proxies_per_validator: JSON.stringify(getProxiesPerValidator()), - } -} - -function getTestnetNetworkVars(celoEnv: string): TerraformVars { - return { - ...getEnvVarValues(testnetNetworkEnvVars), - network_name: networkName(celoEnv), - } -} - -function getEnvVarValues(terraformEnvVars: TerraformVars) { - const vars: { [key: string]: string } = {} - for (const key of Object.keys(terraformEnvVars)) { - vars[key] = fetchEnv(terraformEnvVars[key]) - } - return vars -} - -export async function generateAndUploadSecrets(celoEnv: string) { - // Bootnode - const bootnodeSecrets = generateBootnodeSecretEnvVars() - await uploadSecrets(celoEnv, bootnodeSecrets, 'bootnode') - // Tx Nodes - const txNodeCount = parseInt(fetchEnv(envVar.TX_NODES), 10) - for (let i = 0; i < txNodeCount; i++) { - const secrets = generateNodeSecretEnvVars(AccountType.TX_NODE, i) - await uploadSecrets(celoEnv, secrets, `tx-node-${i}`) - } - // Private tx Nodes - const privateTxNodeCount = parseInt(fetchEnv(envVar.PRIVATE_TX_NODES), 10) - for (let i = 0; i < privateTxNodeCount; i++) { - // Ensure there is no overlap with tx node keys - const secrets = generateNodeSecretEnvVars(AccountType.TX_NODE, i, 1000 + i) - await uploadSecrets(celoEnv, secrets, `tx-node-private-${i}`) - } - // Validators - const validatorCount = parseInt(fetchEnv(envVar.VALIDATORS), 10) - for (let i = 0; i < validatorCount; i++) { - const secrets = generateNodeSecretEnvVars(AccountType.VALIDATOR, i) - await uploadSecrets(celoEnv, secrets, `validator-${i}`) - } - // Proxies - const proxiesPerValidator = getProxiesPerValidator() - let validatorIndex = 0 - for (const proxyCount of proxiesPerValidator) { - for (let i = 0; i < proxyCount; i++) { - const secrets = generateProxySecretEnvVars(validatorIndex, i) - await uploadSecrets(celoEnv, secrets, `validator-${validatorIndex}-proxy-${i}`) - } - validatorIndex++ - } -} - -function uploadSecrets(celoEnv: string, secrets: string, resourceName: string) { - const cloudStorageFileName = `${secretsBasePath(celoEnv)}/.env.${resourceName}` - return uploadDataToGoogleStorage( - secrets, - secretsBucketName(), - cloudStorageFileName, - false, - 'text/plain' - ) -} - -function generateBootnodeSecretEnvVars() { - const mnemonic = fetchEnv(envVar.MNEMONIC) - return formatEnvVars({ - NODE_KEY: generatePrivateKey(mnemonic, AccountType.BOOTNODE, 0), - }) -} - -function generateNodeSecretEnvVars( - accountType: AccountType, - index: number, - keyIndex: number = index -) { - const mnemonic = fetchEnv(envVar.MNEMONIC) - const privateKey = generatePrivateKey(mnemonic, accountType, keyIndex) - const secrets = getNodeSecrets(privateKey) - // If this is meant to be a proxied validator, also generate the enode of its proxy - if (accountType === AccountType.VALIDATOR) { - const proxiesPerValidator = getProxiesPerValidator() - if (index < proxiesPerValidator.length) { - const proxyEnodeAddresses = [] - for (let proxyIndex = 0; proxyIndex < proxiesPerValidator[index]; proxyIndex++) { - proxyEnodeAddresses.push(privateKeyToPublicKey(generateProxyPrivateKey(index, proxyIndex))) - } - secrets.PROXY_ENODE_ADDRESSES = proxyEnodeAddresses.join(',') - } - } - return formatEnvVars(secrets) -} - -function generateProxySecretEnvVars(validatorIndex: number, proxyIndex: number) { - const privateKey = generateProxyPrivateKey(validatorIndex, proxyIndex) - const secrets = getNodeSecrets(privateKey) - secrets.PROXIED_VALIDATOR_ADDRESS = getAddressFromEnv(AccountType.VALIDATOR, validatorIndex) - return formatEnvVars(secrets) -} - -function generateProxyPrivateKey(validatorIndex: number, proxyIndex: number) { - const mnemonic = fetchEnv(envVar.MNEMONIC) - // To allow a validator to have many proxies and to be able to easily - // adjust the number of proxies it has, the following index is calculated - const index = validatorIndex * 10000 + proxyIndex - return generatePrivateKey(mnemonic, AccountType.PROXY, index) -} - -function getNodeSecrets(privateKey: string): NodeSecrets { - const mnemonic = fetchEnv(envVar.MNEMONIC) - return { - ACCOUNT_ADDRESS: privateKeyToAddress(privateKey), - BOOTNODE_ENODE_ADDRESS: generatePublicKey(mnemonic, AccountType.BOOTNODE, 0), - PRIVATE_KEY: privateKey, - [envVar.GETH_ACCOUNT_SECRET]: fetchEnv(envVar.GETH_ACCOUNT_SECRET), - [envVar.MNEMONIC]: mnemonic, - } -} - -// Formats an object into a multi-line string with each line as KEY=VALUE -function formatEnvVars(envVars: { [key: string]: any }) { - return Object.keys(envVars) - .map((key) => `${key}='${envVars[key]}'`) - .join('\n') -} - -function secretsBasePath(celoEnv: string) { - return `vm/${celoEnv}` -} - -function useDefaultNetwork() { - return ( - fetchEnvOrFallback(envVar.VM_BASED, 'false') !== 'true' || - fetchEnv(envVar.KUBERNETES_CLUSTER_NAME) === 'celo-networks-dev' - ) -} - -export function networkName(celoEnv: string) { - return useDefaultNetwork() ? 'default' : `${celoEnv}-network` -} - -function secretsBucketName() { - const config = configForProject() - return config.secretsBucketName -} - -function stateBucketName() { - const config = configForProject() - return config.stateBucketName -} - -function configForProject() { - const project = fetchEnv(envVar.TESTNET_PROJECT_NAME) - if (!projectConfig.hasOwnProperty(project)) { - throw new Error(`No config for project ${project}`) - } - // @ts-ignore - we check above to see if the property exists - return projectConfig[project] -} - -// name of the DNS zone in Google Cloud for a particular domain -function dnsZoneName(domain: string) { - return `${domain}-org` -} - -export function getVmSshCommand(instanceName: string) { - const project = fetchEnv(envVar.TESTNET_PROJECT_NAME) - const zone = fetchEnv(envVar.KUBERNETES_CLUSTER_ZONE) - return `gcloud beta compute --project '${project}' ssh --zone '${zone}' ${instanceName} --tunnel-through-iap` -} - -export function getInternalTxNodeLoadBalancerIpCommand(celoEnv: string) { - const project = fetchEnv(envVar.TESTNET_PROJECT_NAME) - return `gcloud compute forwarding-rules list --project '${project}' --filter="name~'${celoEnv}-tx-node-lb-internal-fwd-rule'" --format='get(IPAddress)'` -} - -export async function getNodeVmName( - celoEnv: string, - nodeType: string, - index?: number | ProxyIndex -) { - const nodeTypesWithRandomSuffixes = ['tx-node', 'tx-node-private', 'proxy'] - const nodeTypesWithNoIndex = ['bootnode'] - let instanceName - if (nodeTypesWithRandomSuffixes.includes(nodeType)) { - instanceName = await getNodeVmNameWithRandomSuffix(celoEnv, nodeType, index || 0) - } else { - instanceName = `${celoEnv}-${nodeType}` - if (!nodeTypesWithNoIndex.includes(nodeType) && index !== undefined) { - instanceName += `-${index}` - } - } - return instanceName -} - -// Some VM names have a randomly generated suffix. This returns the full name -// of the instance given only the celoEnv and index. -async function getNodeVmNameWithRandomSuffix( - celoEnv: string, - nodeType: string, - index: number | ProxyIndex -) { - const project = fetchEnv(envVar.TESTNET_PROJECT_NAME) - - const baseName = - typeof index === 'number' - ? `${celoEnv}-${nodeType}-${index}` - : getProxyName(celoEnv, index.validatorIndex, index.proxyIndex) - - const [nodeName] = await execCmd( - `gcloud compute instances list --project '${project}' --filter="NAME ~ ${baseName}-.*" --format get\\(NAME\\)` - ) - return nodeName.trim() -} - -// indexCoercer is a yargs coercer that parses numeric indices and colon-separated -// indices (:) into a ProxyIndex type. -export function indexCoercer(value: string) { - if (!value) { - return value - } - const splitValues = value.split(':').filter((v) => v) - // Then it's just a single index number - if (splitValues.length === 1) { - return parseInt(value, 10) - } else if (splitValues.length === 2) { - const parsedValues = splitValues.map((v) => parseInt(v, 10)) - const proxyIndex: ProxyIndex = { - validatorIndex: parsedValues[0], - proxyIndex: parsedValues[1], - } - return proxyIndex - } else { - throw new Error('Incorrect index') - } -} diff --git a/packages/terraform-modules/.gitignore b/packages/terraform-modules/.gitignore deleted file mode 100644 index 42b565799f5..00000000000 --- a/packages/terraform-modules/.gitignore +++ /dev/null @@ -1 +0,0 @@ -plan/ diff --git a/packages/terraform-modules/README.md b/packages/terraform-modules/README.md deleted file mode 100644 index 91b921d1f57..00000000000 --- a/packages/terraform-modules/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Terraform Testnets - -## Overview - -Terraform is a tool that allows developers to treat "infrastructure as code." -Infrastructure is defined in modules, and Terraform creates/changes/destroys -when changes are applied. - -## Local Setup - -It's best to use this package with `celotool`, but if you need to -run `terraform` commands locally: - -1. Download Terraform https://www.terraform.io/downloads.html -1. Ensure you have a service account key file at the path shown in the - module's `provider "google"` section. -1. `terraform init` to download anything specific to the module (i.e. GCP specific - binaries) -1. `terraform apply` to initially deploy or upgrade -1. `terraform destroy` to destroy diff --git a/packages/terraform-modules/forno/main.tf b/packages/terraform-modules/forno/main.tf deleted file mode 100644 index 641b7ed2206..00000000000 --- a/packages/terraform-modules/forno/main.tf +++ /dev/null @@ -1,176 +0,0 @@ -# For managing terraform state remotely -terraform { - backend "gcs" { - bucket = "celo_tf_state" - } - required_providers { - google = { - source = "hashicorp/google" - version = "3.69.0" - } - google-beta = { - source = "hashicorp/google-beta" - version = "3.69.0" - } - random = { - source = "hashicorp/random" - version = "3.1.0" - } - } -} - -provider "google" { - credentials = file(var.gcloud_credentials_path) - project = var.gcloud_project - region = "us-west1" - zone = "us-west1-a" -} - -provider "google-beta" { - credentials = file(var.gcloud_credentials_path) - project = var.gcloud_project - region = "us-west1" - zone = "us-west1-a" -} - -data "terraform_remote_state" "state" { - backend = "gcs" - config = { - bucket = "celo_tf_state" - prefix = "${var.celo_env}/forno" - } -} - -module "http_backends" { - source = "./modules/backends" - # variables - backend_max_requests_per_second = var.backend_max_requests_per_second - celo_env = var.celo_env - context_info = var.context_info_http - health_check_destination_port = 6000 - type = "http" - timeout_sec = 60 # 1 minute - security_policy_id = google_compute_security_policy.forno.self_link -} - -module "ws_backends" { - source = "./modules/backends" - # variables - backend_max_requests_per_second = var.backend_max_requests_per_second - celo_env = var.celo_env - context_info = var.context_info_ws - health_check_destination_port = 6001 - type = "ws" - timeout_sec = 1200 # 20 minutes - security_policy_id = google_compute_security_policy.forno.self_link -} - -module "kong" { - source = "./modules/backends" - # variables - backend_max_requests_per_second = var.backend_max_requests_per_second_kong - celo_env = var.celo_env - context_info = var.context_info_kong - health_check_destination_port = 8000 - health_check_request_path = "/kong/status" - type = "kong" - timeout_sec = 60 # 1 minute - security_policy_id = google_compute_security_policy.forno.self_link -} - -resource "google_compute_global_address" "global_address" { - name = "${var.celo_env}-forno-global-address" - - address_type = "EXTERNAL" - ip_version = "IPV4" -} - -resource "google_compute_managed_ssl_certificate" "ssl_cert" { - provider = google-beta - - name = "${var.celo_env}-forno-ssl-cert-${random_id.ssl_random_suffix.hex}" - - managed { - domains = var.ssl_cert_domains - } - - lifecycle { - create_before_destroy = true - } -} - -resource "random_id" "ssl_random_suffix" { - byte_length = 4 - - keepers = { - domains = join(",", var.ssl_cert_domains) - } -} - -resource "google_compute_url_map" "url_map" { - name = "${var.celo_env}-forno-url-map" - default_service = module.kong.backend_service_id - - host_rule { - hosts = ["*"] - path_matcher = "${var.celo_env}-forno-path-matcher" - } - - path_matcher { - name = "${var.celo_env}-forno-path-matcher" - default_service = module.kong.backend_service_id - - path_rule { - paths = ["/ws"] - service = module.ws_backends.backend_service_id - route_action { - url_rewrite { - path_prefix_rewrite = "/" - } - } - } - - path_rule { - paths = ["/kong", "/kong/*"] - service = module.kong.backend_service_id - } - - path_rule { - paths = ["/kong", "/kong/*"] - service = module.kong.backend_service_id - } - } -} - -# This will route ingress traffic to the geographically closest backend -# whose utilization is not full. -# See https://cloud.google.com/load-balancing/docs/https#network-service-tiers_1 -resource "google_compute_target_https_proxy" "target_https_proxy" { - name = "${var.celo_env}-forno-target-https-proxy" - url_map = google_compute_url_map.url_map.id - ssl_certificates = [ - google_compute_managed_ssl_certificate.ssl_cert.id, - ] -} - -resource "google_compute_global_forwarding_rule" "forwarding_rule" { - name = "${var.celo_env}-forno-forwarding-rule" - - target = google_compute_target_https_proxy.target_https_proxy.id - ip_address = google_compute_global_address.global_address.address - port_range = "443" -} - -# This allows GCP health check traffic AND traffic that is being sent from LBs -# to network endpoints -resource "google_compute_firewall" "allow-health-check" { - name = "${var.celo_env}-forno-health-check-firewall" - direction = "INGRESS" - source_ranges = ["130.211.0.0/22", "35.191.0.0/16"] - network = var.vpc_network_name - - allow { - protocol = "tcp" - ports = ["6000", "6001", "8000", "8545", "8546"] - } -} diff --git a/packages/terraform-modules/forno/modules/backends/main.tf b/packages/terraform-modules/forno/modules/backends/main.tf deleted file mode 100644 index 6b99e5bd963..00000000000 --- a/packages/terraform-modules/forno/modules/backends/main.tf +++ /dev/null @@ -1,46 +0,0 @@ -resource "google_compute_health_check" "http_health_check" { - name = "${var.celo_env}-forno-http-health-check-${var.type}" - - http_health_check { - port = var.health_check_destination_port - # For NetworkEndpointGroup, the port specified for each network endpoint is used for health checking - port_specification = "USE_FIXED_PORT" - request_path = var.health_check_request_path - } -} - -# This is a reference to the ClusterIP service inside this region's k8s cluster. -# We get the NEG for each context. -data "google_compute_network_endpoint_group" "service_network_endpoint_group" { - name = each.value.service_network_endpoint_group_name - zone = each.value.zone - - for_each = var.context_info -} - -# A backend that can route traffic to all of the context NEGs. -resource "google_compute_backend_service" "backend_service" { - provider = google-beta - name = "${var.celo_env}-forno-backend-service-${var.type}" - - health_checks = [google_compute_health_check.http_health_check.self_link] - timeout_sec = var.timeout_sec - - custom_response_headers = [ - "Access-Control-Allow-Origin:*", - "Access-Control-Allow-Methods:GET, POST, OPTIONS", - "Access-Control-Allow-Headers:DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range", - "Access-Control-Expose-Headers:Content-Length,Content-Range" - ] - - security_policy = var.security_policy_id - - dynamic "backend" { - for_each = var.context_info - content { - balancing_mode = "RATE" - max_rate_per_endpoint = var.backend_max_requests_per_second - group = data.google_compute_network_endpoint_group.service_network_endpoint_group[backend.key].self_link - } - } -} diff --git a/packages/terraform-modules/forno/modules/backends/outputs.tf b/packages/terraform-modules/forno/modules/backends/outputs.tf deleted file mode 100644 index c26ef7ffbb4..00000000000 --- a/packages/terraform-modules/forno/modules/backends/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "backend_service_id" { - value = google_compute_backend_service.backend_service.id -} diff --git a/packages/terraform-modules/forno/modules/backends/variables.tf b/packages/terraform-modules/forno/modules/backends/variables.tf deleted file mode 100644 index 3435434abf4..00000000000 --- a/packages/terraform-modules/forno/modules/backends/variables.tf +++ /dev/null @@ -1,46 +0,0 @@ -variable "backend_max_requests_per_second" { - type = number - description = "The max number of requests per second that a backend can receive. In this case, a backend refers to all the nodes in a cluster." -} - -variable "celo_env" { - type = string - description = "Name of the Celo environment" -} - -variable "context_info" { - type = map( - object({ - zone = string - service_network_endpoint_group_name = string - }) - ) - description = "Provides basic information on each context. Keys are contexts and values are the corresponding info" -} - -variable "health_check_destination_port" { - type = number - description = "The destination port the health check will test" -} - -variable "health_check_request_path" { - type = string - description = "The requested path the health check will test" - default = "/" -} - -variable "timeout_sec" { - type = number - description = "The timeout for the backend service in seconds" - default = 30 -} - -variable "type" { - type = string - description = "Type of backends, only used for names" -} - -variable "security_policy_id" { - type = string - description = "Cloud Armon security policy ID applied to the backend" -} diff --git a/packages/terraform-modules/forno/outputs.tf b/packages/terraform-modules/forno/outputs.tf deleted file mode 100644 index 88a414bb6c6..00000000000 --- a/packages/terraform-modules/forno/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "forno_ip_address" { - value = google_compute_global_address.global_address.address -} diff --git a/packages/terraform-modules/forno/security_policy.tf b/packages/terraform-modules/forno/security_policy.tf deleted file mode 100644 index 85fd61cc824..00000000000 --- a/packages/terraform-modules/forno/security_policy.tf +++ /dev/null @@ -1,27 +0,0 @@ -resource "google_compute_security_policy" "forno" { - name = "${var.celo_env}-forno-security-policy" - - rule { - action = "deny(403)" - priority = "1000" - match { - versioned_expr = "SRC_IPS_V1" - config { - src_ip_ranges = var.banned_cidr - } - } - description = "Deny access to forno due to unfair usage" - } - - rule { - action = "allow" - priority = "2147483647" - match { - versioned_expr = "SRC_IPS_V1" - config { - src_ip_ranges = ["*"] - } - } - description = "default rule" - } -} diff --git a/packages/terraform-modules/forno/variables.tf b/packages/terraform-modules/forno/variables.tf deleted file mode 100644 index ebd7760f413..00000000000 --- a/packages/terraform-modules/forno/variables.tf +++ /dev/null @@ -1,69 +0,0 @@ -variable "backend_max_requests_per_second" { - type = number - description = "The max number of requests per second that a backend can receive. In this case, a backend refers to each endpoint (pod)" -} - -variable "backend_max_requests_per_second_kong" { - type = number - description = "The max number of requests per second that a backend can receive. In this case, a backend refers to each endpoint (pod)" -} - -variable "celo_env" { - type = string - description = "Name of the Celo environment" -} - -variable "context_info_http" { - type = map( - object({ - zone = string - service_network_endpoint_group_name = string - }) - ) - description = "Provides basic information on each context for HTTP. Keys are contexts and values are the corresponding info" -} - -variable "context_info_ws" { - type = map( - object({ - zone = string - service_network_endpoint_group_name = string - }) - ) - description = "Provides basic information on each context for WS. Keys are contexts and values are the corresponding info" -} - -variable "context_info_kong" { - type = map( - object({ - zone = string - service_network_endpoint_group_name = string - }) - ) - description = "Provides basic information on each context for Kong. Keys are contexts and values are the corresponding info" -} - -variable "gcloud_credentials_path" { - type = string - description = "Path to the file containing the Google Cloud credentials to use" -} - -variable "gcloud_project" { - type = string - description = "Name of the Google Cloud project to use" -} - -variable "ssl_cert_domains" { - type = list(string) - description = "Domains to use for the SSL certificate. Each must end with a period." -} - -variable "banned_cidr" { - type = list(string) - description = "Banned CIDR to make request to forno." -} - -variable "vpc_network_name" { - type = string - description = "The name of the VPC network" -} diff --git a/packages/terraform-modules/forno/versions.tf b/packages/terraform-modules/forno/versions.tf deleted file mode 100644 index 6b6318def82..00000000000 --- a/packages/terraform-modules/forno/versions.tf +++ /dev/null @@ -1,3 +0,0 @@ -terraform { - required_version = ">= 0.13" -} diff --git a/packages/terraform-modules/testnet-network/README.md b/packages/terraform-modules/testnet-network/README.md deleted file mode 100644 index 80a457d255e..00000000000 --- a/packages/terraform-modules/testnet-network/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# testnet-network - -This Terraform module exists as a measure of safely creating and destroying -VPC networks. Because `testnet` may sometimes be on the `default` VPC network -that is used by other resources, we need to make sure that `terraform destroy`ing -the testnet module will not result in the default VPC network being destroyed. -Rather than declaring the network as a `resource` in `testnet`, we instead declare -it as a `data` source. This prevents the network from being deleted upon `terraform destroy`, -but also prevents the network from being created. Terraform lacks basic -conditionals to directly implement this logic in the `testnet` module. - -This module is only intended to be used by `celotool`, which only creates/destroys -a network if it is not the `default` VPC. diff --git a/packages/terraform-modules/testnet-network/main.tf b/packages/terraform-modules/testnet-network/main.tf deleted file mode 100644 index fa7a8daaa68..00000000000 --- a/packages/terraform-modules/testnet-network/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -provider "google" { - credentials = file(var.gcloud_credentials_path) - project = var.gcloud_project - region = "us-west1" - zone = "us-west1-a" -} - -# For managing terraform state remotely -terraform { - backend "gcs" { - bucket = "celo_tf_state" - } -} - -data "terraform_remote_state" "state" { - backend = "gcs" - config = { - bucket = "celo_tf_state" - prefix = "${var.celo_env}/testnet-network" - } -} - -resource "google_compute_network" "testnet-network" { - name = var.network_name -} diff --git a/packages/terraform-modules/testnet-network/variables.tf b/packages/terraform-modules/testnet-network/variables.tf deleted file mode 100644 index 17dc776ca3a..00000000000 --- a/packages/terraform-modules/testnet-network/variables.tf +++ /dev/null @@ -1,19 +0,0 @@ -variable celo_env { - type = string - description = "Name of the testnet Celo environment" -} - -variable gcloud_credentials_path { - type = string - description = "Path to the file containing the Google Cloud credentials to use" -} - -variable gcloud_project { - type = string - description = "Name of the Google Cloud project to use" -} - -variable network_name { - type = string - description = "The name of the network to use" -} diff --git a/packages/terraform-modules/testnet/README.md b/packages/terraform-modules/testnet/README.md deleted file mode 100644 index f7df4fd7114..00000000000 --- a/packages/terraform-modules/testnet/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Testnet - -This creates a VM-based testnet with a bootnode and multiple validators. - -## Overview - -Each type of node (validator/bootnode/tx-node) is in its own module. -A separate module `tx-node-load-balancer` defines an internal TCP load balancer -for ports 8545 and 8546. This is so Blockscout on the same VPC can reach the tx-nodes. -The GCP provider, network, firewall etc declarations are found in the `main.tf` file. - -The script that is run immediately upon the startup of a VM instance is found in -a module's `startup.sh` file. This is where `geth` or `bootnode` is started, -and any setup work is performed. The variables required by these are typically -pulled from a `.env` file by `celotool` and passed to `terraform`. - -Sometimes, if recreating an address right after deleting one, GCP will say -that the resource already exists and `terraform apply` will fail. In this case, -just wait a little bit and try again. - -## Setup - -A few infrastructure assumptions are made by this module and `celotool`: - -1. Google Cloud Platform is used -1. The environment variable `GOOGLE_APPLICATION_CREDENTIALS` is set to the path - of a JSON key file with the credentials that will be used by Terraform. -1. The GCP project used is the value of `TESTNET_PROJECT_NAME` in the environment's `.env` file -1. A bucket called `celo_tf_state` has been manually created. This is where the Terraform - remote backends are. Nothing very sensitive is stored in here, but it should be private. -1. A bucket called `celo-testnet-secrets` has been manually created. This is used to store - sensitive secrets that are uploaded by celotool and downloaded at startup from inside - `startup.sh`. This is to ensure that secrets are not stored in the state files. - -## Google Cloud Permissions Needed - -A service account must be able to create/list/modify/delete networks, -firewalls, instances, objects, addresses, and disks. - -For cLabs employees, a Google Cloud role `Terraform Testnet Admin` has been -created. diff --git a/packages/terraform-modules/testnet/main.tf b/packages/terraform-modules/testnet/main.tf deleted file mode 100644 index 33c035ab94f..00000000000 --- a/packages/terraform-modules/testnet/main.tf +++ /dev/null @@ -1,242 +0,0 @@ -provider "google" { - credentials = file(var.gcloud_credentials_path) - project = var.gcloud_project - region = "us-west1" - zone = "us-west1-a" -} - -provider "acme" { - server_url = "https://acme-v02.api.letsencrypt.org/directory" -} - -# For managing terraform state remotely -terraform { - backend "gcs" { - bucket = "celo_tf_state" - } - required_providers { - google = "~> 2.16.0" - } -} - -data "terraform_remote_state" "state" { - backend = "gcs" - config = { - bucket = "celo_tf_state" - prefix = "${var.celo_env}/testnet" - } -} - -locals { - target_tag_bootnode = "${var.celo_env}-bootnode" - # any geth node (tx nodes & validators) - target_tag_node = "${var.celo_env}-node" - - target_tag_proxy = "${var.celo_env}-proxy" - target_tag_tx_node = "${var.celo_env}-tx-node" - target_tag_tx_node_private = "${var.celo_env}-tx-node-private" - target_tag_validator = "${var.celo_env}-validator" - - target_tag_ssl = "${var.celo_env}-external-ssl" - - target_tags_all = [ - local.target_tag_bootnode, - local.target_tag_node, - local.target_tag_proxy, - local.target_tag_ssl - ] -} - -data "google_compute_network" "network" { - name = var.network_name -} - -resource "google_compute_firewall" "ssh_firewall" { - name = "${var.celo_env}-ssh-firewall" - network = data.google_compute_network.network.name - - target_tags = local.target_tags_all - - allow { - protocol = "tcp" - ports = ["22"] - } -} - -resource "google_compute_firewall" "geth_firewall" { - name = "${var.celo_env}-geth-firewall" - network = data.google_compute_network.network.name - - target_tags = [local.target_tag_node] - - allow { - protocol = "tcp" - ports = ["30303"] - } - - allow { - protocol = "udp" - ports = ["30303"] - } -} - -resource "google_compute_firewall" "geth_metrics_firewall" { - name = "${var.celo_env}-geth-metrics-firewall" - network = data.google_compute_network.network.name - - target_tags = [local.target_tag_node] - - # allow all IPs internal to the VPC - source_ranges = ["10.0.0.0/8"] - - allow { - protocol = "tcp" - ports = ["6060", "9200"] - } -} - -resource "google_compute_firewall" "rpc_firewall_internal" { - name = "${var.celo_env}-rpc-firewall-internal" - network = data.google_compute_network.network.name - - target_tags = [local.target_tag_tx_node_private] - - # allow all IPs internal to the VPC - source_ranges = ["10.0.0.0/8"] - - allow { - protocol = "tcp" - ports = ["8545", "8546"] - } -} - -resource "google_compute_firewall" "rpc_firewall" { - name = "${var.celo_env}-rpc-firewall" - network = data.google_compute_network.network.name - - target_tags = [local.target_tag_tx_node] - - allow { - protocol = "tcp" - ports = ["8545", "8546"] - } -} - -resource "google_compute_firewall" "bootnode_firewall" { - name = "${var.celo_env}-bootnode-firewall" - network = data.google_compute_network.network.name - - target_tags = [local.target_tag_bootnode] - - allow { - protocol = "udp" - ports = ["30301"] - } -} - -module "bootnode" { - source = "./modules/bootnode" - # variables - celo_env = var.celo_env - gcloud_secrets_base_path = var.gcloud_secrets_base_path - gcloud_secrets_bucket = var.gcloud_secrets_bucket - gcloud_vm_service_account_email = var.gcloud_vm_service_account_email - geth_bootnode_docker_image_repository = var.geth_bootnode_docker_image_repository - geth_bootnode_docker_image_tag = var.geth_bootnode_docker_image_tag - network_id = var.network_id - network_name = data.google_compute_network.network.name -} - -module "tx_node" { - source = "./modules/full-node" - # variables - block_time = var.block_time - bootnode_ip_address = module.bootnode.ip_address - celo_env = var.celo_env - ethstats_host = var.ethstats_host - gcloud_secrets_base_path = var.gcloud_secrets_base_path - gcloud_secrets_bucket = var.gcloud_secrets_bucket - gcloud_vm_service_account_email = var.gcloud_vm_service_account_email - genesis_content_base64 = var.genesis_content_base64 - geth_metrics = var.geth_metrics - geth_node_docker_image_repository = var.geth_node_docker_image_repository - geth_node_docker_image_tag = var.geth_node_docker_image_tag - geth_verbosity = var.geth_verbosity - in_memory_discovery_table = var.in_memory_discovery_table - instance_tags = [local.target_tag_tx_node] - max_peers = 500 - name = "tx-node" - network_id = var.network_id - network_name = data.google_compute_network.network.name - gcmode = "full" - node_count = var.tx_node_count - node_disk_size_gb = var.node_disk_size_gb - rpc_apis = "eth,net,web3" -} - -module "tx_node_private" { - source = "./modules/full-node" - # variables - block_time = var.block_time - bootnode_ip_address = module.bootnode.ip_address - celo_env = var.celo_env - ethstats_host = var.ethstats_host - gcloud_secrets_base_path = var.gcloud_secrets_base_path - gcloud_secrets_bucket = var.gcloud_secrets_bucket - gcloud_vm_service_account_email = var.gcloud_vm_service_account_email - genesis_content_base64 = var.genesis_content_base64 - geth_metrics = var.geth_metrics - geth_node_docker_image_repository = var.geth_node_docker_image_repository - geth_node_docker_image_tag = var.geth_node_docker_image_tag - geth_verbosity = var.geth_verbosity - in_memory_discovery_table = var.in_memory_discovery_table - instance_tags = [local.target_tag_tx_node_private] - max_peers = 500 - name = "tx-node-private" - network_id = var.network_id - network_name = data.google_compute_network.network.name - gcmode = "archive" - node_count = var.private_tx_node_count - rpc_apis = "eth,net,web3,debug,txpool" - node_disk_size_gb = var.private_node_disk_size_gb -} - -# used for access by blockscout -module "tx_node_lb" { - source = "./modules/tx-node-load-balancer" - # variables - celo_env = var.celo_env - dns_gcloud_project = var.dns_gcloud_project - dns_zone_name = var.dns_zone_name - forno_host = var.forno_host - gcloud_project = var.gcloud_project - gcloud_vm_service_account_email = var.gcloud_vm_service_account_email - letsencrypt_email = var.letsencrypt_email - network_name = data.google_compute_network.network.name - private_tx_node_self_links = module.tx_node_private.self_links - tx_node_self_links = module.tx_node.self_links -} - -module "validator" { - source = "./modules/validator" - # variables - block_time = var.block_time - bootnode_ip_address = module.bootnode.ip_address - celo_env = var.celo_env - ethstats_host = var.ethstats_host - gcloud_secrets_base_path = var.gcloud_secrets_base_path - gcloud_secrets_bucket = var.gcloud_secrets_bucket - gcloud_vm_service_account_email = var.gcloud_vm_service_account_email - genesis_content_base64 = var.genesis_content_base64 - geth_metrics = var.geth_metrics - geth_node_docker_image_repository = var.geth_node_docker_image_repository - geth_node_docker_image_tag = var.geth_node_docker_image_tag - geth_verbosity = var.geth_verbosity - in_memory_discovery_table = var.in_memory_discovery_table - istanbul_request_timeout_ms = var.istanbul_request_timeout_ms - network_id = var.network_id - network_name = data.google_compute_network.network.name - node_disk_size_gb = var.node_disk_size_gb - proxies_per_validator = var.proxies_per_validator - validator_count = var.validator_count -} diff --git a/packages/terraform-modules/testnet/modules/bootnode/main.tf b/packages/terraform-modules/testnet/modules/bootnode/main.tf deleted file mode 100644 index 777615df7dd..00000000000 --- a/packages/terraform-modules/testnet/modules/bootnode/main.tf +++ /dev/null @@ -1,54 +0,0 @@ -locals { - name_prefix = "${var.celo_env}-bootnode" -} - -resource "google_compute_address" "bootnode" { - name = "${local.name_prefix}-address" - address_type = "EXTERNAL" -} - -resource "google_compute_instance" "bootnode" { - name = local.name_prefix - machine_type = "n1-standard-1" - - tags = [local.name_prefix] - - allow_stopping_for_update = true - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - scratch_disk { - - } - - network_interface { - network = var.network_name - access_config { - nat_ip = google_compute_address.bootnode.address - } - } - - metadata_startup_script = templatefile( - format("%s/startup.sh", path.module), { - gcloud_secrets_base_path : var.gcloud_secrets_base_path, - gcloud_secrets_bucket : var.gcloud_secrets_bucket, - geth_bootnode_docker_image_repository : var.geth_bootnode_docker_image_repository, - geth_bootnode_docker_image_tag : var.geth_bootnode_docker_image_tag, - ip_address : google_compute_address.bootnode.address, - network_id : var.network_id - } - ) - - service_account { - email = var.gcloud_vm_service_account_email - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ] - } -} diff --git a/packages/terraform-modules/testnet/modules/bootnode/outputs.tf b/packages/terraform-modules/testnet/modules/bootnode/outputs.tf deleted file mode 100644 index edc627de1c3..00000000000 --- a/packages/terraform-modules/testnet/modules/bootnode/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output ip_address { - value = google_compute_address.bootnode.address -} diff --git a/packages/terraform-modules/testnet/modules/bootnode/startup.sh b/packages/terraform-modules/testnet/modules/bootnode/startup.sh deleted file mode 100644 index 2a3a08aa0c5..00000000000 --- a/packages/terraform-modules/testnet/modules/bootnode/startup.sh +++ /dev/null @@ -1,149 +0,0 @@ -#! /bin/bash - -# ---- Set Up Logging ---- - -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -bash install-logging-agent.sh - -echo " -@include config.d/*.conf -# Prometheus monitoring. - - @type prometheus - port 24231 - - - @type prometheus_monitor - - -# Do not collect fluentd's own logs to avoid infinite loops. - - @type null - - -# Add a unique insertId to each log entry that doesn't already have it. -# This helps guarantee the order and prevent log duplication. - -@type add_insert_ids - - -# Configure all sources to output to Google Cloud Logging - - @type google_cloud - buffer_type file - buffer_path /var/log/google-fluentd/buffers - # Set the chunk limit conservatively to avoid exceeding the recommended - # chunk size of 5MB per write request. - buffer_chunk_limit 512KB - # Flush logs every 5 seconds, even if the buffer is not full. - flush_interval 5s - # Enforce some limit on the number of retries. - disable_retry_limit false - # After 3 retries, a given chunk will be discarded. - retry_limit 3 - # Wait 10 seconds before the first retry. The wait interval will be doubled on - # each following retry (20s, 40s...) until it hits the retry limit. - retry_wait 10 - # Never wait longer than 5 minutes between retries. If the wait interval - # reaches this limit, the exponentiation stops. - # Given the default config, this limit should never be reached, but if - # retry_limit and retry_wait are customized, this limit might take effect. - max_retry_wait 300 - # Use multiple threads for processing. - num_threads 8 - # Use the gRPC transport. - use_grpc true - # If a request is a mix of valid log entries and invalid ones, ingest the - # valid ones and drop the invalid ones instead of dropping everything. - partial_success true - # Enable monitoring via Prometheus integration. - enable_monitoring true - monitoring_type opencensus - detect_json true -" > /etc/google-fluentd/google-fluentd.conf - -echo " - - @type rewrite_tag_filter - - key log - pattern ^{ - tag docker_logs_json - - - key log - pattern ^[^{] - tag docker_logs_plain - - - - - @type parser - key_name log - reserve_data false - - @type json - - - - - @type record_transformer - - message $${record["log"]} - - -" > /etc/google-fluentd/config.d/docker.conf -systemctl restart google-fluentd - -# ---- Set Up Monitoring Agent ---- - -curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh -bash install-monitoring-agent.sh - -# ---- Install Docker ---- - -echo "Installing Docker..." - -# TODO(trevor): investigate how to pull this into a separate file so -# other startup scripts can use it -apt update && apt upgrade -apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 -curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" -apt update && apt upgrade -apt install -y docker-ce -systemctl start docker - -echo "Configuring Docker..." -gcloud auth configure-docker - -# use GCP logging for Docker containers -echo '{"log-driver":"gcplogs"}' > /etc/docker/daemon.json -systemctl restart docker - -# ---- Set Up and Run Geth ---- - -BOOTNODE_VERBOSITY=1 - -GETH_BOOTNODE_DOCKER_IMAGE=${geth_bootnode_docker_image_repository}:${geth_bootnode_docker_image_tag} - -# download & apply secrets pulled from Cloud Storage as environment vars -echo "Downloading secrets from Google Cloud Storage..." -SECRETS_ENV_PATH=/var/.env.celo.secrets -gsutil cp gs://${gcloud_secrets_bucket}/${gcloud_secrets_base_path}/.env.bootnode $SECRETS_ENV_PATH -# Apply the .env file -. $SECRETS_ENV_PATH - -echo "Pulling bootnode..." -docker pull $GETH_BOOTNODE_DOCKER_IMAGE - -echo "Starting bootnode..." -docker run -p 30301:30301/udp --name bootnode --net=host --restart=always -d $GETH_BOOTNODE_DOCKER_IMAGE /bin/sh -c "\ - set -euo pipefail && \ - mkdir /etc/bootnode && \ - echo $NODE_KEY > /etc/bootnode/node.key && \ - /usr/local/bin/bootnode \ - --nat=extip:${ip_address} \ - --networkid=${network_id} \ - --nodekey=/etc/bootnode/node.key \ - --verbosity=$BOOTNODE_VERBOSITY" diff --git a/packages/terraform-modules/testnet/modules/bootnode/variables.tf b/packages/terraform-modules/testnet/modules/bootnode/variables.tf deleted file mode 100644 index 6493eca4dc2..00000000000 --- a/packages/terraform-modules/testnet/modules/bootnode/variables.tf +++ /dev/null @@ -1,39 +0,0 @@ -variable celo_env { - type = string - description = "Name of the testnet Celo environment" -} - -variable gcloud_secrets_base_path { - type = string - description = "Base path in the secrets bucket of a Google Cloud Storage file containing validator secrets" -} - -variable gcloud_secrets_bucket { - type = string - description = "Name of the Google Cloud Storage bucket where secrets are kept" -} - -variable gcloud_vm_service_account_email { - type = string - description = "The email of the service account to associate virtual machines with" -} - -variable geth_bootnode_docker_image_repository { - type = string - description = "Repository of the geth bootnode docker image" -} - -variable geth_bootnode_docker_image_tag { - type = string - description = "Tag of the geth bootnode docker image" -} - -variable network_id { - type = number - description = "The network ID number" -} - -variable network_name { - type = string - description = "Name of the GCP network" -} diff --git a/packages/terraform-modules/testnet/modules/full-node/main.tf b/packages/terraform-modules/testnet/modules/full-node/main.tf deleted file mode 100644 index 3414589ac8c..00000000000 --- a/packages/terraform-modules/testnet/modules/full-node/main.tf +++ /dev/null @@ -1,116 +0,0 @@ -locals { - attached_disk_name = "celo-data" - name_prefix = "${var.celo_env}-${var.name}" - # generate names using `var.name` if `var.names` isn't set - names = length(var.names) > 0 ? var.names : [for node_index in range(var.node_count) : "${var.name}-${node_index}"] -} - -resource "google_compute_address" "full_node" { - name = "${var.celo_env}-${each.key}-address-${random_id.full_node[each.key].hex}" - address_type = "EXTERNAL" - - for_each = local.names - - lifecycle { - create_before_destroy = true - } -} - -resource "google_compute_instance" "full_node" { - name = "${var.celo_env}-${each.key}-${random_id.full_node[each.key].hex}" - machine_type = "n1-standard-2" - - for_each = local.names - - tags = concat(["${var.celo_env}-node"], var.instance_tags) - - allow_stopping_for_update = true - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - attached_disk { - source = google_compute_disk.full_node[each.key].self_link - device_name = local.attached_disk_name - } - - network_interface { - network = var.network_name - access_config { - nat_ip = google_compute_address.full_node[each.key].address - } - } - - metadata_startup_script = templatefile( - format("%s/startup.sh", path.module), { - additional_geth_flags : var.additional_geth_flags, - attached_disk_name : local.attached_disk_name, - block_time : var.block_time, - bootnode_ip_address : var.bootnode_ip_address, - ethstats_host : var.ethstats_host, - gcloud_secrets_base_path : var.gcloud_secrets_base_path, - gcloud_secrets_bucket : var.gcloud_secrets_bucket, - genesis_content_base64 : var.genesis_content_base64, - geth_metrics : var.geth_metrics, - geth_node_docker_image_repository : var.geth_node_docker_image_repository, - geth_node_docker_image_tag : var.geth_node_docker_image_tag, - geth_verbosity : var.geth_verbosity, - in_memory_discovery_table : var.in_memory_discovery_table, - ip_address : google_compute_address.full_node[each.key].address, - max_light_peers : var.max_light_peers, - max_peers : var.max_peers, - name : each.key, - network_id : var.network_id, - network_name : var.celo_env, - gcmode: var.gcmode, - node_name : "${var.celo_env}-${each.key}", - proxy : var.proxy, - rid : each.key, - rpc_apis : var.rpc_apis, - } - ) - - service_account { - email = var.gcloud_vm_service_account_email - scopes = [ - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ] - } - - lifecycle { - create_before_destroy = true - } -} - -resource "google_compute_disk" "full_node" { - name = "${var.celo_env}-${each.key}-disk-${random_id.full_node_disk[each.key].hex}" - - for_each = local.names - - type = "pd-ssd" - # in GB - size = var.node_disk_size_gb - physical_block_size_bytes = 4096 - - lifecycle { - create_before_destroy = true - } -} - -resource "random_id" "full_node" { - for_each = local.names - - byte_length = 8 -} - -# Separate random id so that updating the ID of the instance doesn't force a new disk -resource "random_id" "full_node_disk" { - for_each = local.names - - byte_length = 8 -} diff --git a/packages/terraform-modules/testnet/modules/full-node/outputs.tf b/packages/terraform-modules/testnet/modules/full-node/outputs.tf deleted file mode 100644 index 7f8141af2de..00000000000 --- a/packages/terraform-modules/testnet/modules/full-node/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output ip_addresses { - value = [for v in google_compute_address.full_node : v.address] -} - -output internal_ip_addresses { - value = [for v in google_compute_instance.full_node : v.network_interface.0.network_ip] -} - -output self_links { - value = [for v in google_compute_instance.full_node : v.self_link] -} - -output ip_addresses_map { - value = { for k, v in google_compute_address.full_node : k => v.address } -} - -output internal_ip_addresses_map { - value = { for k, v in google_compute_instance.full_node : k => v.network_interface.0.network_ip } -} diff --git a/packages/terraform-modules/testnet/modules/full-node/startup.sh b/packages/terraform-modules/testnet/modules/full-node/startup.sh deleted file mode 100644 index 6c661bb4b94..00000000000 --- a/packages/terraform-modules/testnet/modules/full-node/startup.sh +++ /dev/null @@ -1,241 +0,0 @@ -#! /bin/bash - -# ---- Set Up Logging ---- - -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -bash install-logging-agent.sh - -echo " -@include config.d/*.conf -# Prometheus monitoring. - - @type prometheus - port 24231 - - - @type prometheus_monitor - - -# Do not collect fluentd's own logs to avoid infinite loops. - - @type null - - -# Add a unique insertId to each log entry that doesn't already have it. -# This helps guarantee the order and prevent log duplication. - -@type add_insert_ids - - -# Configure all sources to output to Google Cloud Logging - - @type google_cloud - buffer_type file - buffer_path /var/log/google-fluentd/buffers - # Set the chunk limit conservatively to avoid exceeding the recommended - # chunk size of 5MB per write request. - buffer_chunk_limit 512KB - # Flush logs every 5 seconds, even if the buffer is not full. - flush_interval 5s - # Enforce some limit on the number of retries. - disable_retry_limit false - # After 3 retries, a given chunk will be discarded. - retry_limit 3 - # Wait 10 seconds before the first retry. The wait interval will be doubled on - # each following retry (20s, 40s...) until it hits the retry limit. - retry_wait 10 - # Never wait longer than 5 minutes between retries. If the wait interval - # reaches this limit, the exponentiation stops. - # Given the default config, this limit should never be reached, but if - # retry_limit and retry_wait are customized, this limit might take effect. - max_retry_wait 300 - # Use multiple threads for processing. - num_threads 8 - # Use the gRPC transport. - use_grpc true - # If a request is a mix of valid log entries and invalid ones, ingest the - # valid ones and drop the invalid ones instead of dropping everything. - partial_success true - # Enable monitoring via Prometheus integration. - enable_monitoring true - monitoring_type opencensus - detect_json true -" > /etc/google-fluentd/google-fluentd.conf - -echo " - - @type rewrite_tag_filter - - key log - pattern ^{ - tag docker_logs_json - - - key log - pattern ^[^{] - tag docker_logs_plain - - - - - @type parser - key_name log - reserve_data false - - @type json - - - - - @type record_transformer - - message $${record["log"]} - - -" > /etc/google-fluentd/config.d/docker.conf -systemctl restart google-fluentd - -# ---- Set Up Monitoring Agent ---- - -curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh -bash install-monitoring-agent.sh - -# ---- Set Up Persistent Disk ---- - -# gives a path similar to `/dev/sdb` -DISK_PATH=`readlink -f /dev/disk/by-id/google-${attached_disk_name}` -DATA_DIR=/root/.celo - -echo "Setting up persistent disk ${attached_disk_name} at $DISK_PATH..." - -DISK_FORMAT=ext4 -CURRENT_DISK_FORMAT=`lsblk -i -n -o fstype $DISK_PATH` - -echo "Checking if disk $DISK_PATH format $CURRENT_DISK_FORMAT matches desired $DISK_FORMAT..." - -# If the disk has already been formatted previously (this will happen -# if this instance has been recreated with the same disk), we skip formatting -if [[ $CURRENT_DISK_FORMAT == $DISK_FORMAT ]]; then - echo "Disk $DISK_PATH is correctly formatted as $DISK_FORMAT" -else - echo "Disk $DISK_PATH is not formatted correctly, formatting as $DISK_FORMAT..." - mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard $DISK_PATH -fi - -mkdir -p $DATA_DIR -echo "Mounting $DISK_PATH onto $DATA_DIR" -mount -o discard,defaults $DISK_PATH $DATA_DIR - -# ---- Install Docker ---- - -echo "Installing Docker..." -apt update && apt upgrade -apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 -curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" -apt update && apt upgrade -apt install -y docker-ce -systemctl start docker - -echo "Configuring Docker..." -gcloud auth configure-docker - -# use GCP logging for Docker containers -echo '{"log-driver":"fluentd","log-opts":{"fluentd-address":"0.0.0.0:24224","tag":"docker_logs"}}' > /etc/docker/daemon.json -systemctl restart docker - -# ---- Set Up and Run Geth ---- - -GETH_NODE_DOCKER_IMAGE=${geth_node_docker_image_repository}:${geth_node_docker_image_tag} - -# download & apply secrets pulled from Cloud Storage as environment vars -echo "Downloading secrets from Google Cloud Storage..." -SECRETS_ENV_PATH=/var/.env.celo.secrets -gsutil cp gs://${gcloud_secrets_bucket}/${gcloud_secrets_base_path}/.env.${name} $SECRETS_ENV_PATH -# Apply the .env file -. $SECRETS_ENV_PATH - -echo "Address: $ACCOUNT_ADDRESS" -echo "Bootnode enode address: $BOOTNODE_ENODE_ADDRESS" - -BOOTNODE_ENODE=$BOOTNODE_ENODE_ADDRESS@${bootnode_ip_address}:30301 -echo "Bootnode enode: $BOOTNODE_ENODE" - -echo "Pulling geth..." -docker pull $GETH_NODE_DOCKER_IMAGE - -IN_MEMORY_DISCOVERY_TABLE_FLAG="" -[[ ${in_memory_discovery_table} == "true" ]] && IN_MEMORY_DISCOVERY_TABLE_FLAG="--use-in-memory-discovery-table" - -RPC_APIS=${rpc_apis} - -if [[ ${proxy} == "true" ]]; then - ADDITIONAL_GETH_FLAGS="--proxy.proxy --proxy.internalendpoint :30503 --proxy.proxiedvalidatoraddress $PROXIED_VALIDATOR_ADDRESS" -fi - -METRICS_FLAGS="" -if [[ ${geth_metrics} == "true" ]]; then - # Valid from celo-blockchain >=1.5.x - METRICS_FLAGS="$METRICS_FLAGS --metrics --pprof --pprof.port 6060 --pprof.addr 127.0.0.1" -fi - -# Using bootnode so their enode url will be published to the discv5 DHT (and light clients can discover them) -BOOTNODE_FLAG="--bootnodes=enode://$BOOTNODE_ENODE" - -DATA_DIR=/root/.celo - -mkdir -p $DATA_DIR/account -echo -n "${genesis_content_base64}" | base64 -d > $DATA_DIR/genesis.json -echo -n "${rid}" > $DATA_DIR/replica_id -echo -n "${ip_address}" > $DATA_DIR/ipAddress -echo -n "$PRIVATE_KEY" > $DATA_DIR/pkey -echo -n "$ACCOUNT_ADDRESS" > $DATA_DIR/address -echo -n "$BOOTNODE_ENODE_ADDRESS" > $DATA_DIR/bootnodeEnodeAddress -echo -n "$BOOTNODE_ENODE" > $DATA_DIR/bootnodeEnode -echo -n "$GETH_ACCOUNT_SECRET" > $DATA_DIR/account/accountSecret - -echo "Starting geth..." -# We need to override the entrypoint in the geth image (which is originally `geth`) -docker run \ - -v $DATA_DIR:$DATA_DIR \ - --name geth \ - --net=host \ - --restart always \ - --entrypoint /bin/sh \ - -d \ - $GETH_NODE_DOCKER_IMAGE -c "\ - ( - set -euo pipefail ; \ - geth init $DATA_DIR/genesis.json \ - ) ; \ - geth account import --password $DATA_DIR/account/accountSecret $DATA_DIR/pkey ; \ - geth \ - --$BOOTNODE_FLAG \ - --datadir $DATA_DIR \ - --light.serve 90 \ - --light.maxpeers ${max_light_peers} \ - --maxpeers=${max_peers} \ - --nousb \ - --rpc \ - --rpcaddr 0.0.0.0 \ - --rpcapi=$RPC_APIS \ - --rpccorsdomain='*' \ - --rpcvhosts=* \ - --ws \ - --wsaddr 0.0.0.0 \ - --wsorigins=* \ - --wsapi=$RPC_APIS \ - --nodekey=$DATA_DIR/pkey \ - --etherbase=$ACCOUNT_ADDRESS \ - --networkid=${network_id} \ - --syncmode=full \ - --gcmode=${gcmode} \ - --consoleformat=json \ - --consoleoutput=stdout \ - --verbosity=${geth_verbosity} \ - --ethstats=${node_name}@${ethstats_host} \ - --metrics \ - --pprof \ - $IN_MEMORY_DISCOVERY_TABLE_FLAG \ - $ADDITIONAL_GETH_FLAGS" - diff --git a/packages/terraform-modules/testnet/modules/full-node/variables.tf b/packages/terraform-modules/testnet/modules/full-node/variables.tf deleted file mode 100644 index 0dda2aca227..00000000000 --- a/packages/terraform-modules/testnet/modules/full-node/variables.tf +++ /dev/null @@ -1,139 +0,0 @@ -variable additional_geth_flags { - type = string - description = "Additional flags to be passed when running geth" - default = "" -} - -variable block_time { - type = number - description = "Number of seconds between each block" -} - -variable bootnode_ip_address { - type = string - description = "The external IP address of the bootnode" -} - -variable celo_env { - type = string - description = "Name of the testnet Celo environment" -} - -variable ethstats_host { - type = string - description = "Ethstats url or IP address" -} - -variable gcloud_secrets_base_path { - type = string - description = "Base path in the secrets bucket of a Google Cloud Storage file containing tx-node secrets" -} - -variable gcloud_secrets_bucket { - type = string - description = "Name of the Google Cloud Storage bucket where secrets are kept" -} - -variable gcloud_vm_service_account_email { - type = string - description = "The email of the service account to associate virtual machines with" -} - -variable genesis_content_base64 { - type = string - description = "Content of the genesis file encoded in base64" -} - -variable geth_metrics { - type = string - description = "Enable Geth metrics (prometheus format) on port 6060" -} - -variable geth_node_docker_image_repository { - type = string - description = "Repository of the geth docker image" -} - -variable geth_node_docker_image_tag { - type = string - description = "Tag of the geth docker image" -} - -variable geth_verbosity { - type = number - description = "Verbosity of the nodes" -} - -variable in_memory_discovery_table { - type = bool - description = "Specifies whether to use an in memory discovery table" -} - -variable instance_tags { - type = list(string) - description = "Tags to set for the instance" - default = [] -} - -variable max_light_peers { - type = number - description = "The maximum number of light client peers" - default = 50 -} - -variable max_peers { - type = number - description = "The maximum number of peers for the node" - default = 100 -} - -variable name { - type = string - description = "Name of the nodes. Should be specified if names is not." - default = "" -} - -variable names { - type = set(string) - description = "Name of each node to create. If not specified, the names will be generated using the name variable and an index." - default = [] -} - -variable network_id { - type = number - description = "The network ID number" -} - -variable network_name { - type = string - description = "Name of the GCP network the node VM is in" -} - -variable node_count { - type = number - description = "Number of nodes to create if names is not specified" - default = 0 -} - -variable node_disk_size_gb { - type = number - description = "The size in GB for each node's disk" -} - -variable gcmode { - type = string - description = "Celo-blockchain --gcmode option" - default = "full" -} - -variable proxy { - type = bool - description = "Whether the node is a proxy for a validator" - default = false -} - -variable rpc_apis { - type = string - description = "Comma separated string including which RPC APIs to expose" - default = "eth,net,web3" -} diff --git a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/main.tf b/packages/terraform-modules/testnet/modules/tx-node-load-balancer/main.tf deleted file mode 100644 index db760f315b7..00000000000 --- a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/main.tf +++ /dev/null @@ -1,244 +0,0 @@ -locals { - name_prefix = "${var.celo_env}-tx-node-lb" - target_https_proxy_name = "${var.celo_env}-tx-node-lb-external-http-proxy" -} - -# We want to maintain websockets (which are not supposed by the HTTPS external -# load balancer) & avoid unnecessary egress costs. -# An internal & external load balancer cannot use the same instance group. To -# get around this, we allocate 1 of the tx-nodes to be for internal load balancing. -# It's still included in `static_nodes.json`, but not included in the forno -# setup. In the future, consider moving this node to live in Kubernetes to be -# along with the services that use it. - -# internal load balancer for cLabs-run infra: - -resource "google_compute_instance_group" "internal" { - name = "${local.name_prefix}-internal-group-${random_id.internal.hex}" - - instances = var.private_tx_node_self_links - - lifecycle { - create_before_destroy = true - } -} - -resource "random_id" "internal" { - byte_length = 8 -} - -data "google_compute_subnetwork" "subnet" { - name = var.network_name -} - -resource "google_compute_address" "internal" { - name = "${local.name_prefix}-internal-address" - address_type = "INTERNAL" - subnetwork = data.google_compute_subnetwork.subnet.self_link -} - -resource "google_compute_forwarding_rule" "internal" { - name = "${local.name_prefix}-internal-fwd-rule" - - backend_service = google_compute_region_backend_service.internal.self_link - ip_address = google_compute_address.internal.address - load_balancing_scheme = "INTERNAL" - network = var.network_name - ports = ["8545", "8546"] -} - -resource "google_compute_region_backend_service" "internal" { - name = "${local.name_prefix}-internal-service" - - # internal HTTP load balancing does not support WebSockets - protocol = "TCP" - - backend { - group = google_compute_instance_group.internal.self_link - } - - health_checks = [ - google_compute_health_check.internal.self_link - ] -} - -resource "google_compute_health_check" "internal" { - name = "${local.name_prefix}-internal-health" - - tcp_health_check { - port = 8545 - } -} - -# external load balancer for forno setup - - -resource "google_compute_instance_group" "external" { - name = "${local.name_prefix}-group-${random_id.external.hex}" - - instances = var.tx_node_self_links - - lifecycle { - create_before_destroy = true - } - - named_port { - name = "http" - port = "8545" - } -} - -resource "random_id" "external" { - byte_length = 8 -} - -resource "google_compute_global_address" "external" { - name = "${local.name_prefix}-external-address" - address_type = "EXTERNAL" -} - -resource "google_compute_global_forwarding_rule" "external" { - name = "${local.name_prefix}-external-fwd-rule" - - ip_address = google_compute_global_address.external.address - load_balancing_scheme = "EXTERNAL" - port_range = "443" - target = google_compute_target_https_proxy.external.self_link -} - -resource "google_compute_target_https_proxy" "external" { - name = "${local.name_prefix}-external-http-proxy" - url_map = google_compute_url_map.external.self_link - ssl_certificates = [google_compute_ssl_certificate.external.self_link] - quic_override = "NONE" -} - -resource "google_compute_url_map" "external" { - name = "${local.name_prefix}-external-url-map" - default_service = google_compute_backend_service.external.self_link - - host_rule { - hosts = [var.forno_host] - path_matcher = "allpaths" - } - - path_matcher { - name = "allpaths" - default_service = google_compute_backend_service.external.self_link - } -} - -resource "google_compute_backend_service" "external" { - name = "${local.name_prefix}-external-service" - port_name = "http" - protocol = "HTTP" - - backend { - group = google_compute_instance_group.external.self_link - } - - health_checks = [ - google_compute_health_check.external.self_link - ] -} - -resource "google_compute_health_check" "external" { - name = "${local.name_prefix}-external-health" - - http_health_check { - port = 8545 - } -} - -resource "google_dns_record_set" "external" { - # google cloud requires the name to end with a "." - name = "${var.forno_host}." - managed_zone = data.google_dns_managed_zone.external.name - type = "A" - ttl = 3600 - - rrdatas = [google_compute_global_address.external.address] - - project = var.dns_gcloud_project -} - -data "google_dns_managed_zone" "external" { - name = var.dns_zone_name - project = var.dns_gcloud_project -} - -# SSL certificate from Let's Encrypt: - -resource "google_compute_instance" "external_ssl" { - name = "${local.name_prefix}-external-ssl" - machine_type = "f1-micro" - - tags = ["${var.celo_env}-external-ssl"] - - allow_stopping_for_update = true - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - network_interface { - network = var.network_name - access_config { - } - } - - metadata_startup_script = templatefile( - format("%s/ssl-startup.sh", path.module), { - cert_prefix : "${local.name_prefix}-forno-", - forno_host : var.forno_host, - gcloud_project : var.dns_gcloud_project, - letsencrypt_email : var.letsencrypt_email, - target_https_proxy_name : local.target_https_proxy_name - } - ) - - service_account { - email = var.gcloud_vm_service_account_email - scopes = [ - "https://www.googleapis.com/auth/compute", - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/ndev.clouddns.readwrite" - ] - } -} - -# temporary self-signed certificate that will be overwritten by -# google_compute_instance.external_ssl - -resource "google_compute_ssl_certificate" "external" { - name_prefix = "${local.name_prefix}-ssl-cert" - private_key = tls_private_key.tmp.private_key_pem - certificate = tls_self_signed_cert.tmp.cert_pem - - lifecycle { - create_before_destroy = true - } -} - -resource "tls_self_signed_cert" "tmp" { - key_algorithm = "RSA" - private_key_pem = tls_private_key.tmp.private_key_pem - - subject { - common_name = var.forno_host - organization = "Temporary self signed cert" - } - - validity_period_hours = 12 - - allowed_uses = [ - "server_auth", - ] -} - -resource "tls_private_key" "tmp" { - algorithm = "RSA" -} diff --git a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/output.tf b/packages/terraform-modules/testnet/modules/tx-node-load-balancer/output.tf deleted file mode 100644 index aba2eac8be9..00000000000 --- a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/output.tf +++ /dev/null @@ -1,3 +0,0 @@ -output internal_ip_address { - value = google_compute_address.internal.address -} diff --git a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/ssl-startup.sh b/packages/terraform-modules/testnet/modules/tx-node-load-balancer/ssl-startup.sh deleted file mode 100644 index e4e674aa3e3..00000000000 --- a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/ssl-startup.sh +++ /dev/null @@ -1,129 +0,0 @@ -#! /bin/bash - -# ---- Set Up Logging ---- - -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -bash install-logging-agent.sh - -echo " -@include config.d/*.conf -# Prometheus monitoring. - - @type prometheus - port 24231 - - - @type prometheus_monitor - - -# Do not collect fluentd's own logs to avoid infinite loops. - - @type null - - -# Add a unique insertId to each log entry that doesn't already have it. -# This helps guarantee the order and prevent log duplication. - -@type add_insert_ids - - -# Configure all sources to output to Google Cloud Logging - - @type google_cloud - buffer_type file - buffer_path /var/log/google-fluentd/buffers - # Set the chunk limit conservatively to avoid exceeding the recommended - # chunk size of 5MB per write request. - buffer_chunk_limit 512KB - # Flush logs every 5 seconds, even if the buffer is not full. - flush_interval 5s - # Enforce some limit on the number of retries. - disable_retry_limit false - # After 3 retries, a given chunk will be discarded. - retry_limit 3 - # Wait 10 seconds before the first retry. The wait interval will be doubled on - # each following retry (20s, 40s...) until it hits the retry limit. - retry_wait 10 - # Never wait longer than 5 minutes between retries. If the wait interval - # reaches this limit, the exponentiation stops. - # Given the default config, this limit should never be reached, but if - # retry_limit and retry_wait are customized, this limit might take effect. - max_retry_wait 300 - # Use multiple threads for processing. - num_threads 8 - # Use the gRPC transport. - use_grpc true - # If a request is a mix of valid log entries and invalid ones, ingest the - # valid ones and drop the invalid ones instead of dropping everything. - partial_success true - # Enable monitoring via Prometheus integration. - enable_monitoring true - monitoring_type opencensus - detect_json true -" > /etc/google-fluentd/google-fluentd.conf - -echo " - - @type rewrite_tag_filter - - key log - pattern ^{ - tag docker_logs_json - - - key log - pattern ^[^{] - tag docker_logs_plain - - - - - @type parser - key_name log - reserve_data false - - @type json - - - - - @type record_transformer - - message $${record["log"]} - - -" > /etc/google-fluentd/config.d/docker.conf -systemctl restart google-fluentd - -# ---- Install Docker ---- - -echo "Installing Docker..." -apt update && apt upgrade -apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 -curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" -apt update && apt upgrade -apt install -y docker-ce -systemctl start docker - -echo "Configuring Docker..." -gcloud auth configure-docker - -# use GCP logging for Docker containers -echo '{"log-driver":"gcplogs"}' > /etc/docker/daemon.json -systemctl restart docker - -mkdir -p /home/lego - -# use --env USE_STAGING_SERVER=true to test staging - -/usr/bin/docker run -d \ - -v /home/lego:/root/.lego \ - --restart always \ - --env GCE_PROJECT=${gcloud_project} \ - --env LETSENCRYPT_EMAIL=${letsencrypt_email} \ - --env TARGET_PROXY=${target_https_proxy_name} \ - --env DOMAINS_LIST="-d ${forno_host}" \ - --env CERT_ID_PREFIX=${cert_prefix} \ - --name=ssl-letsencrypt \ - bloomapi/letsencrypt-gcloud-balancer:v1.0.2 diff --git a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/variables.tf b/packages/terraform-modules/testnet/modules/tx-node-load-balancer/variables.tf deleted file mode 100644 index bfe2bb19dbe..00000000000 --- a/packages/terraform-modules/testnet/modules/tx-node-load-balancer/variables.tf +++ /dev/null @@ -1,49 +0,0 @@ -variable celo_env { - type = string - description = "Name of the testnet Celo environment" -} - -variable dns_gcloud_project { - type = string - description = "Name of the Google Cloud project where Cloud DNS is" -} - -variable dns_zone_name { - type = string - description = "Name of the DNS zone for the domain used for the forno setup" -} - -variable forno_host { - type = string - description = "The host name to use for the tx node forno setup" -} - -variable gcloud_project { - type = string - description = "Name of the Google Cloud project to use" -} - -variable gcloud_vm_service_account_email { - type = string - description = "The email of the service account to associate virtual machines with" -} - -variable letsencrypt_email { - type = string - description = "The email to create letsencrypt certificates with" -} - -variable network_name { - type = string - description = "Name of the GCP network the tx-node load balancer is in" -} - -variable private_tx_node_self_links { - type = list(string) - description = "A list including the self_links of each private/internal tx-node" -} - -variable tx_node_self_links { - type = list(string) - description = "A list including the self_links of each public/external tx-node" -} diff --git a/packages/terraform-modules/testnet/modules/validator/main.tf b/packages/terraform-modules/testnet/modules/validator/main.tf deleted file mode 100644 index 36ed6958b35..00000000000 --- a/packages/terraform-modules/testnet/modules/validator/main.tf +++ /dev/null @@ -1,191 +0,0 @@ -# This module creates var.validator_count validators. The first -# local.proxied_validator_count validators are hidden behind externally facing -# proxies, and the rest are exposed to the external internet. - -locals { - attached_disk_name = "celo-data" - name_prefix = "${var.celo_env}-validator" - proxied_validator_count = length(var.proxies_per_validator) -} - -resource "google_compute_address" "validator" { - name = "${local.name_prefix}-address-${count.index}" - address_type = "EXTERNAL" - - # only create external addresses for validators that are not proxied - count = var.validator_count - local.proxied_validator_count -} - -resource "google_compute_address" "validator_internal" { - name = "${local.name_prefix}-internal-address-${count.index}" - subnetwork = google_compute_subnetwork.validator.self_link - address_type = "INTERNAL" - purpose = "GCE_ENDPOINT" - - count = var.validator_count -} - -resource "google_compute_instance" "validator" { - name = "${local.name_prefix}-${count.index}" - machine_type = "n1-standard-2" - - count = var.validator_count - - tags = ["${var.celo_env}-node", "${var.celo_env}-validator"] - - allow_stopping_for_update = true - - boot_disk { - initialize_params { - image = "debian-cloud/debian-9" - } - } - - attached_disk { - source = google_compute_disk.validator[count.index].self_link - device_name = local.attached_disk_name - } - - network_interface { - network = var.network_name - network_ip = google_compute_address.validator_internal[count.index].address - subnetwork = google_compute_subnetwork.validator.name - # We only want an access config for validators that will not be proxied - dynamic "access_config" { - for_each = count.index < local.proxied_validator_count ? [] : [0] - content { - nat_ip = google_compute_address.validator[count.index - local.proxied_validator_count].address - } - } - } - - metadata_startup_script = templatefile( - format("%s/startup.sh", path.module), { - attached_disk_name : local.attached_disk_name, - block_time : var.block_time, - bootnode_ip_address : var.bootnode_ip_address, - ethstats_host : var.ethstats_host, - gcloud_secrets_base_path : var.gcloud_secrets_base_path, - gcloud_secrets_bucket : var.gcloud_secrets_bucket, - genesis_content_base64 : var.genesis_content_base64, - geth_metrics : var.geth_metrics, - geth_node_docker_image_repository : var.geth_node_docker_image_repository, - geth_node_docker_image_tag : var.geth_node_docker_image_tag, - geth_verbosity : var.geth_verbosity, - in_memory_discovery_table : var.in_memory_discovery_table, - ip_address : count.index < local.proxied_validator_count ? "" : google_compute_address.validator[count.index - local.proxied_validator_count].address, - istanbul_request_timeout_ms : var.istanbul_request_timeout_ms, - max_light_peers : 20, - max_peers : 125, - network_id : var.network_id, - network_name : var.celo_env, - proxied : count.index < length(var.proxies_per_validator), - # proxied : var.proxies_per_validator[count.index] > 0 ? true : false, - rid : count.index, - # Searches for all proxies whose map key corresponds to this specific validator - # by finding keys starting with "validator-${this validator index}" - proxy_internal_ip_addresses : compact([for key in keys(module.proxy.internal_ip_addresses_map) : substr(key, 0, length(format("validator-%d", count.index))) == format("validator-%d", count.index) ? module.proxy.internal_ip_addresses_map[key] : ""]), - proxy_external_ip_addresses : compact([for key in keys(module.proxy.ip_addresses_map) : substr(key, 0, length(format("validator-%d", count.index))) == format("validator-%d", count.index) ? module.proxy.ip_addresses_map[key] : ""]), - validator_name : "${local.name_prefix}-${count.index}", - } - ) - - service_account { - email = var.gcloud_vm_service_account_email - scopes = [ - "https://www.googleapis.com/auth/compute", - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ] - } -} - -resource "google_compute_disk" "validator" { - name = "${local.name_prefix}-disk-${count.index}" - count = var.validator_count - - type = "pd-ssd" - # in GB - size = var.node_disk_size_gb - physical_block_size_bytes = 4096 -} - -resource "google_compute_subnetwork" "validator" { - name = "${local.name_prefix}-subnet" - network = var.network_name - # Arbitrary IP range. This cannot overlap with existing subnetwork IP ranges - # in the same network, so there can only be at most 1 VM testnet on a VPC network - ip_cidr_range = "10.25.0.0/24" - # to allow an internal instance to reach google API servers (metrics reporting, logs, etc) - private_ip_google_access = true -} - -# proxies - -module "proxy" { - source = "../full-node" - # variables - block_time = var.block_time - bootnode_ip_address = var.bootnode_ip_address - celo_env = var.celo_env - ethstats_host = var.ethstats_host - gcloud_secrets_base_path = var.gcloud_secrets_base_path - gcloud_secrets_bucket = var.gcloud_secrets_bucket - gcloud_vm_service_account_email = var.gcloud_vm_service_account_email - genesis_content_base64 = var.genesis_content_base64 - geth_metrics = var.geth_metrics - geth_node_docker_image_repository = var.geth_node_docker_image_repository - geth_node_docker_image_tag = var.geth_node_docker_image_tag - geth_verbosity = var.geth_verbosity - in_memory_discovery_table = var.in_memory_discovery_table - instance_tags = ["${var.celo_env}-proxy"] - max_peers = 200 - names = flatten([for val_index in range(length(var.proxies_per_validator)) : [for proxy_index in range(var.proxies_per_validator[val_index]) : format("validator-%d-proxy-%d", val_index, proxy_index)]]) - network_id = var.network_id - network_name = var.network_name - node_disk_size_gb = var.node_disk_size_gb - proxy = true -} - -# if there are no proxied validators, we don't have to worry about - -resource "google_compute_firewall" "proxy_internal_ingress" { - count = local.proxied_validator_count > 0 ? 1 : 0 - - name = "${local.name_prefix}-proxy-internal-ingress" - network = var.network_name - - direction = "INGRESS" - source_ranges = ["10.0.0.0/8"] - - allow { - protocol = "tcp" - ports = ["30503"] - } - - allow { - protocol = "udp" - ports = ["30503"] - } -} - -resource "google_compute_firewall" "proxy_internal_egress" { - count = local.proxied_validator_count > 0 ? 1 : 0 - - name = "${local.name_prefix}-proxy-internal-egress" - network = var.network_name - - direction = "EGRESS" - destination_ranges = ["10.0.0.0/8"] - - allow { - protocol = "tcp" - ports = ["30503"] - } - - allow { - protocol = "udp" - ports = ["30503"] - } -} diff --git a/packages/terraform-modules/testnet/modules/validator/outputs.tf b/packages/terraform-modules/testnet/modules/validator/outputs.tf deleted file mode 100644 index 53c502e9e04..00000000000 --- a/packages/terraform-modules/testnet/modules/validator/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output internal_ip_addresses { - value = google_compute_address.validator_internal.*.address -} - -output proxy_internal_ip_addresses { - value = module.proxy.internal_ip_addresses -} diff --git a/packages/terraform-modules/testnet/modules/validator/startup.sh b/packages/terraform-modules/testnet/modules/validator/startup.sh deleted file mode 100644 index b368f759531..00000000000 --- a/packages/terraform-modules/testnet/modules/validator/startup.sh +++ /dev/null @@ -1,300 +0,0 @@ -#! /bin/bash - -GCLOUD_ZONE=`gcloud compute instances list --filter="name=('${validator_name}')" --format 'value(zone)'` - -# If this validator is proxied, it won't have an access config. We need to -# create one for the initial 1 time setup so we can reach the external internet -if [[ ${proxied} == "true" ]]; then - gcloud compute instances add-access-config ${validator_name} --zone=$GCLOUD_ZONE -fi - -# ---- Set Up Logging ---- - -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -bash install-logging-agent.sh - -echo " -@include config.d/*.conf -# Prometheus monitoring. - - @type prometheus - port 24231 - - - @type prometheus_monitor - - -# Do not collect fluentd's own logs to avoid infinite loops. - - @type null - - -# Add a unique insertId to each log entry that doesn't already have it. -# This helps guarantee the order and prevent log duplication. - -@type add_insert_ids - - -# Configure all sources to output to Google Cloud Logging - - @type google_cloud - buffer_type file - buffer_path /var/log/google-fluentd/buffers - # Set the chunk limit conservatively to avoid exceeding the recommended - # chunk size of 5MB per write request. - buffer_chunk_limit 512KB - # Flush logs every 5 seconds, even if the buffer is not full. - flush_interval 5s - # Enforce some limit on the number of retries. - disable_retry_limit false - # After 3 retries, a given chunk will be discarded. - retry_limit 3 - # Wait 10 seconds before the first retry. The wait interval will be doubled on - # each following retry (20s, 40s...) until it hits the retry limit. - retry_wait 10 - # Never wait longer than 5 minutes between retries. If the wait interval - # reaches this limit, the exponentiation stops. - # Given the default config, this limit should never be reached, but if - # retry_limit and retry_wait are customized, this limit might take effect. - max_retry_wait 300 - # Use multiple threads for processing. - num_threads 8 - # Use the gRPC transport. - use_grpc true - # If a request is a mix of valid log entries and invalid ones, ingest the - # valid ones and drop the invalid ones instead of dropping everything. - partial_success true - # Enable monitoring via Prometheus integration. - enable_monitoring true - monitoring_type opencensus - detect_json true -" > /etc/google-fluentd/google-fluentd.conf - -echo " - - @type rewrite_tag_filter - - key log - pattern ^{ - tag docker_logs_json - - - key log - pattern ^[^{] - tag docker_logs_plain - - - - - @type parser - key_name log - reserve_data false - - @type json - - - - - @type record_transformer - - message $${record["log"]} - - -" > /etc/google-fluentd/config.d/docker.conf -systemctl restart google-fluentd - -# ---- Set Up Monitoring Agent ---- - -curl -sSO https://dl.google.com/cloudagents/install-monitoring-agent.sh -bash install-monitoring-agent.sh - -# ---- Set Up Persistent Disk ---- - -# gives a path similar to `/dev/sdb` -DISK_PATH=`readlink -f /dev/disk/by-id/google-${attached_disk_name}` -DATA_DIR=/root/.celo - -echo "Setting up persistent disk ${attached_disk_name} at $DISK_PATH..." - -DISK_FORMAT=ext4 -CURRENT_DISK_FORMAT=`lsblk -i -n -o fstype $DISK_PATH` - -echo "Checking if disk $DISK_PATH format $CURRENT_DISK_FORMAT matches desired $DISK_FORMAT..." - -# If the disk has already been formatted previously (this will happen -# if this instance has been recreated with the same disk), we skip formatting -if [[ $CURRENT_DISK_FORMAT == $DISK_FORMAT ]]; then - echo "Disk $DISK_PATH is correctly formatted as $DISK_FORMAT" -else - echo "Disk $DISK_PATH is not formatted correctly, formatting as $DISK_FORMAT..." - mkfs.ext4 -m 0 -F -E lazy_itable_init=0,lazy_journal_init=0,discard $DISK_PATH -fi - -mkdir -p $DATA_DIR -echo "Mounting $DISK_PATH onto $DATA_DIR" -mount -o discard,defaults $DISK_PATH $DATA_DIR - -# ---- Install Docker ---- - -echo "Installing Docker..." -apt update && apt upgrade -apt install -y apt-transport-https ca-certificates curl software-properties-common gnupg2 -curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" -apt update && apt upgrade -apt install -y docker-ce -systemctl start docker - -echo "Configuring Docker..." -gcloud auth configure-docker - -# use GCP logging for Docker containers -echo '{"log-driver":"fluentd","log-opts":{"fluentd-address":"0.0.0.0:24224","tag":"docker_logs"}}' > /etc/docker/daemon.json -systemctl restart docker - -# ---- Set Up and Run Geth ---- - -GETH_NODE_DOCKER_IMAGE=${geth_node_docker_image_repository}:${geth_node_docker_image_tag} - -# download & apply secrets pulled from Cloud Storage as environment vars -echo "Downloading secrets from Google Cloud Storage..." -SECRETS_ENV_PATH=/var/.env.celo.secrets -gsutil cp gs://${gcloud_secrets_bucket}/${gcloud_secrets_base_path}/.env.validator-${rid} $SECRETS_ENV_PATH -# Apply the .env file -. $SECRETS_ENV_PATH - -echo "Address: $ACCOUNT_ADDRESS" -echo "Bootnode enode address: $BOOTNODE_ENODE_ADDRESS" - -BOOTNODE_ENODE=$BOOTNODE_ENODE_ADDRESS@${bootnode_ip_address}:30301 -echo "Bootnode enode: $BOOTNODE_ENODE" - -echo "Pulling geth..." -docker pull $GETH_NODE_DOCKER_IMAGE - -PROXIED_FLAGS="" -PROXY_ENODE="" -if [[ ${proxied} == "true" ]]; then - - PROXY_COUNT=${length(proxy_internal_ip_addresses)} - PROXY_INTERNAL_IP_ADDRESSES=${join(",", proxy_internal_ip_addresses)} - PROXY_EXTERNAL_IP_ADDRESSES=${join(",", proxy_external_ip_addresses)} - - PROXY_ENODE_URL_PAIRS="" - - PROXY_INDEX=0 - while [ $PROXY_INDEX -lt $PROXY_COUNT ]; do - PROXY_INTERNAL_IP_ADDRESS=`echo -n $PROXY_INTERNAL_IP_ADDRESSES | cut -d ',' -f $(($PROXY_INDEX + 1))` - PROXY_EXTERNAL_IP_ADDRESS=`echo -n $PROXY_EXTERNAL_IP_ADDRESSES | cut -d ',' -f $(($PROXY_INDEX + 1))` - # $PROXY_ENODE_ADDRESSES is from the secrets pulled from google cloud - PROXY_ENODE_ADDRESS=`echo -n $PROXY_ENODE_ADDRESSES | cut -d ',' -f $(($PROXY_INDEX + 1))` - - PROXY_INTERNAL_ENODE="enode://$PROXY_ENODE_ADDRESS@$PROXY_INTERNAL_IP_ADDRESS:30503" - PROXY_EXTERNAL_ENODE="enode://$PROXY_ENODE_ADDRESS@$PROXY_EXTERNAL_IP_ADDRESS:30303" - - echo "Proxy $PROXY_INDEX internal enode: $PROXY_INTERNAL_ENODE" - echo "Proxy $PROXY_INDEX external enode: $PROXY_EXTERNAL_ENODE" - - if [ $PROXY_INDEX -gt 0 ]; then - PROXY_ENODE_URL_PAIRS="$PROXY_ENODE_URL_PAIRS," - fi - PROXY_ENODE_URL_PAIRS="$PROXY_ENODE_URL_PAIRS$PROXY_INTERNAL_ENODE;$PROXY_EXTERNAL_ENODE" - - PROXY_INDEX=$(($PROXY_INDEX + 1)) - done - if docker run --rm --entrypoint=geth $GETH_NODE_DOCKER_IMAGE --help | grep 'proxy.proxyenodeurlpairs' >/dev/null; then - PROXY_FLAG_NAME="--proxy.proxyenodeurlpairs" - else - PROXY_FLAG_NAME="--proxy.proxyenodeurlpair" - fi - PROXIED_FLAGS="--proxy.proxied --nodiscover $PROXY_FLAG_NAME=\"$PROXY_ENODE_URL_PAIRS\"" - - # if this validator is proxied, cut it off from the external internet after - # we've downloaded everything - echo "Deleting access config" - # The command hangs but still succeeds, give it some time - # This is likely because when the access config is actually deleted, this - # instance cannot reach the external internet so the success ack from the server - # is never received - timeout 20 gcloud compute instances delete-access-config ${validator_name} --zone=$GCLOUD_ZONE -fi - -METRICS_FLAGS="" -if [[ ${geth_metrics} == "true" ]]; then - # Valid from celo-blockchain >=1.5.x - METRICS_FLAGS="$METRICS_FLAGS --metrics --pprof --pprof.port 6060 --pprof.addr 127.0.0.1" -fi - -IN_MEMORY_DISCOVERY_TABLE_FLAG="" -[[ ${in_memory_discovery_table} == "true" ]] && IN_MEMORY_DISCOVERY_TABLE_FLAG="--use-in-memory-discovery-table" - -mkdir -p $DATA_DIR/account -echo -n "${genesis_content_base64}" | base64 -d > $DATA_DIR/genesis.json -echo -n "${rid}" > $DATA_DIR/replica_id -echo -n "$ACCOUNT_ADDRESS" > $DATA_DIR/address -echo -n "$BOOTNODE_ENODE_ADDRESS" > $DATA_DIR/bootnodeEnodeAddress -echo -n "$BOOTNODE_ENODE" > $DATA_DIR/bootnodeEnode -echo -n "$GETH_ACCOUNT_SECRET" > $DATA_DIR/account/accountSecret - -NAT_FLAG="" -if [ "${ip_address}" ]; then - echo -n "${ip_address}" > $DATA_DIR/ipAddress - NAT_FLAG="--nat=extip:${ip_address}" -fi - -if [[ "${network_name}" == "alfajores" || "${network_name}" == "baklava" ]]; then - BOOTNODE_FLAG="--${network_name}" -else - BOOTNODE_FLAG="--bootnodes=enode://$BOOTNODE_ENODE" -fi - -echo "Starting geth..." -# We need to override the entrypoint in the geth image (which is originally `geth`). -# `geth account import` fails when the account has already been imported. In -# this case, we do not want to pipefail -docker run \ - -v $DATA_DIR:$DATA_DIR \ - --name geth \ - --net=host \ - --restart always \ - --entrypoint /bin/sh \ - -d \ - $GETH_NODE_DOCKER_IMAGE -c "\ - ( - set -euo pipefail ; \ - geth init $DATA_DIR/genesis.json \ - ) ; \ - TMP_PRIVATE_KEY_FILE=$(mktemp) ; \ - echo -n $PRIVATE_KEY > \$TMP_PRIVATE_KEY_FILE ; \ - geth account import --password $DATA_DIR/account/accountSecret \$TMP_PRIVATE_KEY_FILE ; \ - rm \$TMP_PRIVATE_KEY_FILE ; \ - geth \ - --$BOOTNODE_FLAG \ - --datadir $DATA_DIR \ - --nousb \ - --password=$DATA_DIR/account/accountSecret \ - --unlock=$ACCOUNT_ADDRESS \ - --mine \ - --rpc \ - --rpcaddr 0.0.0.0 \ - --rpcapi=eth,net,web3 \ - --rpccorsdomain='*' \ - --rpcvhosts=* \ - --ws \ - --wsaddr 0.0.0.0 \ - --wsorigins=* \ - --wsapi=eth,net,web3 \ - --etherbase=$ACCOUNT_ADDRESS \ - --networkid=${network_id} \ - --syncmode=full \ - --consoleformat=json \ - --consoleoutput=stdout \ - --verbosity=${geth_verbosity} \ - --ethstats=${validator_name}@${ethstats_host} \ - --maxpeers=${max_peers} \ - --allow-insecure-unlock \ - $METRICS_FLAGS \ - $NAT_FLAG \ - $IN_MEMORY_DISCOVERY_TABLE_FLAG \ - $PROXIED_FLAGS" - diff --git a/packages/terraform-modules/testnet/modules/validator/variables.tf b/packages/terraform-modules/testnet/modules/validator/variables.tf deleted file mode 100644 index 02251abb4c3..00000000000 --- a/packages/terraform-modules/testnet/modules/validator/variables.tf +++ /dev/null @@ -1,94 +0,0 @@ -variable block_time { - type = number - description = "Number of seconds between each block" -} - -variable bootnode_ip_address { - type = string - description = "The external IP address of the bootnode" -} - -variable celo_env { - type = string - description = "Name of the testnet Celo environment" -} - -variable ethstats_host { - type = string - description = "Ethstats url or IP address" -} - -variable gcloud_secrets_base_path { - type = string - description = "Base path in the secrets bucket of a Google Cloud Storage file containing validator secrets" -} - -variable gcloud_secrets_bucket { - type = string - description = "Name of the Google Cloud Storage bucket where secrets are kept" -} - -variable gcloud_vm_service_account_email { - type = string - description = "The email of the service account to associate virtual machines with" -} - -variable genesis_content_base64 { - type = string - description = "Content of the genesis file encoded in base64" -} - -variable geth_metrics { - type = string - description = "Enable Geth metrics (prometheus format) on port 6060" -} - -variable geth_node_docker_image_repository { - type = string - description = "Repository of the geth docker image" -} - -variable geth_node_docker_image_tag { - type = string - description = "Tag of the geth docker image" -} - -variable geth_verbosity { - type = number - description = "Verbosity of the validator nodes" -} - -variable in_memory_discovery_table { - type = bool - description = "Specifies whether to use an in memory discovery table" -} - -variable istanbul_request_timeout_ms { - type = number - description = "The number of ms for the istanbul request timeout" -} - -variable network_id { - type = number - description = "The network ID number" -} - -variable network_name { - type = string - description = "Name of the GCP network the validator VM is in" -} - -variable node_disk_size_gb { - type = number - description = "The size in GB for each node's disk" -} - -variable proxies_per_validator { - type = list(number) - description = "Number of proxies for each validator that is proxied. Does not include validators that aren't proxied. indices correspond to validator indices." -} - -variable validator_count { - type = number - description = "Number of validators to create" -} diff --git a/packages/terraform-modules/testnet/outputs.tf b/packages/terraform-modules/testnet/outputs.tf deleted file mode 100644 index d2d163423c6..00000000000 --- a/packages/terraform-modules/testnet/outputs.tf +++ /dev/null @@ -1,23 +0,0 @@ -output bootnode_ip_address { - value = module.bootnode.ip_address -} - -output tx_node_internal_ip_addresses { - value = module.tx_node.internal_ip_addresses -} - -output tx_node_ip_addresses { - value = module.tx_node.ip_addresses -} - -output tx_node_lb_internal_ip_address { - value = module.tx_node_lb.internal_ip_address -} - -output validator_internal_ip_addresses { - value = module.validator.internal_ip_addresses -} - -output proxy_internal_ip_addresses { - value = module.validator.proxy_internal_ip_addresses -} diff --git a/packages/terraform-modules/testnet/variables.tf b/packages/terraform-modules/testnet/variables.tf deleted file mode 100644 index b4954b0727b..00000000000 --- a/packages/terraform-modules/testnet/variables.tf +++ /dev/null @@ -1,144 +0,0 @@ -variable block_time { - type = number - description = "Number of seconds between each block" -} - -variable celo_env { - type = string - description = "Name of the testnet Celo environment" -} - -variable dns_gcloud_project { - type = string - description = "Name of the Google Cloud project where Cloud DNS is" -} - -variable dns_zone_name { - type = string - description = "Name of the DNS zone for the domain used for the forno setup" -} - -variable ethstats_host { - type = string - description = "Ethstats url or IP address" -} - -variable forno_host { - type = string - description = "The host name to use for the tx node forno setup" -} - -variable gcloud_credentials_path { - type = string - description = "Path to the file containing the Google Cloud credentials to use" -} - -variable gcloud_project { - type = string - description = "Name of the Google Cloud project to use" -} - -variable gcloud_secrets_base_path { - type = string - description = "Base path in the secrets bucket of a Google Cloud Storage file containing node secrets" -} - -variable gcloud_secrets_bucket { - type = string - description = "Name of the Google Cloud Storage bucket where secrets are kept" -} - -variable gcloud_vm_service_account_email { - type = string - description = "The email of the service account to associate virtual machines with" -} - -variable genesis_content_base64 { - type = string - description = "Content of the genesis file encoded in base64" -} - -variable geth_bootnode_docker_image_repository { - type = string - description = "Repository of the bootnode docker image" -} - -variable geth_bootnode_docker_image_tag { - type = string - description = "Tag of the bootnode docker image" -} - -variable geth_metrics { - type = string - description = "Enable Geth metrics (prometheus format) on port 6060" -} - -variable geth_node_docker_image_repository { - type = string - description = "Repository of the geth docker image" -} - -variable geth_node_docker_image_tag { - type = string - description = "Tag of the geth docker image" -} - -variable geth_verbosity { - type = number - description = "Verbosity of all geth nodes" -} - -variable in_memory_discovery_table { - type = bool - description = "Specifies whether to use an in memory discovery table" -} - -variable istanbul_request_timeout_ms { - type = number - description = "The number of ms for the istanbul request timeout" -} - -variable letsencrypt_email { - type = string - description = "The email to create letsencrypt certificates with" -} - -variable network_id { - type = number - description = "The network ID number" -} - -variable network_name { - type = string - description = "The name of the network to use" -} -variable node_disk_size_gb { - type = number - description = "The size in GB of disks for validators, proxies, and txnodes" -} - -variable private_node_disk_size_gb { - type = number - description = "The size in GB of disks for all private txnodes" -} - -variable private_tx_node_count { - type = number - description = "Number of private tx-nodes that are created with RPC ports only internally exposed" - default = 0 -} - -variable proxies_per_validator { - type = list(number) - description = "Number of proxies for each validator that is proxied. Does not include validators that aren't proxied. indices correspond to validator indices." -} - -variable tx_node_count { - type = number - description = "Number of public tx-nodes to create" -} - -variable validator_count { - type = number - description = "Number of validators to create" -}