From 163d75c3562808e9a0283f2521d60e160863d386 Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Tue, 26 Jan 2021 12:33:04 +0100 Subject: [PATCH 01/14] Bumping versions bump 'min_packer_version' -> 1.5.4 bump 'vault_version' -> 1.6.1 bump 'consul_version' -> 1.9.2 --- examples/vault-consul-ami/vault-consul.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/vault-consul-ami/vault-consul.json b/examples/vault-consul-ami/vault-consul.json index 8a350c36..0ebbac94 100644 --- a/examples/vault-consul-ami/vault-consul.json +++ b/examples/vault-consul-ami/vault-consul.json @@ -1,10 +1,10 @@ { - "min_packer_version": "0.12.0", + "min_packer_version": "1.5.4", "variables": { "aws_region": "us-east-1", - "vault_version": "1.5.4", + "vault_version": "1.6.1", "consul_module_version": "v0.8.0", - "consul_version": "1.5.3", + "consul_version": "1.9.2", "consul_download_url": "{{env `CONSUL_DOWNLOAD_URL`}}", "vault_download_url": "{{env `VAULT_DOWNLOAD_URL`}}", "install_auth_signing_script": "true", From 9cf2eeeb575af690a304c45e70000db0019bbeb8 Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Wed, 27 Jan 2021 22:57:22 +0100 Subject: [PATCH 02/14] Updating vault service configuration Adding 'service_registration' used when using a different storage backend like 's3' Updating systemd configuration based on this documentation https://learn.hashicorp.com/tutorials/vault/deployment-guide\#step-3-configure-systemd --- modules/run-vault/run-vault | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index c7982409..0c17fb90 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -288,6 +288,7 @@ EOF local dynamodb_storage_type="storage" local s3_config="" local vault_storage_backend="" + local service_registration="" if [[ "$enable_s3_backend" == "true" ]]; then s3_config=$(cat <> "$config_path" echo -e "$s3_config" >> "$config_path" echo -e "$vault_storage_backend" >> "$config_path" + echo -e "$service_registration" >> "$config_path" chown "$user:$user" "$config_path" } @@ -368,6 +375,8 @@ Documentation=https://www.vaultproject.io/docs/ Requires=network-online.target After=network-online.target ConditionFileNotEmpty=$config_path +StartLimitIntervalSec=60 +StartLimitBurst=3 EOF ) @@ -392,9 +401,12 @@ KillSignal=SIGINT Restart=on-failure RestartSec=5 TimeoutStopSec=30 +StartLimitInterval=60 StartLimitIntervalSec=60 StartLimitBurst=3 LimitNOFILE=65536 +LimitMEMLOCK=infinity + EOF ) From c43a74c1f8f571bf7a6ef1349df2e1713c35fb08 Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Wed, 27 Jan 2021 23:02:01 +0100 Subject: [PATCH 03/14] Refactoring tests Add a new test stage 'initialize_unseal' useful when running multiple times the 'validation' stage against the same cluster Add 'testVaultUsesConsulForDns' for all test cases --- test/vault_cluster_private_test.go | 11 ++++- test/vault_cluster_public_test.go | 12 +++++- test/vault_cluster_s3_backend_test.go | 14 +++--- test/vault_helpers.go | 62 ++++++++++++++++++++++++++- test/vault_main_test.go | 1 + 5 files changed, 89 insertions(+), 11 deletions(-) diff --git a/test/vault_cluster_private_test.go b/test/vault_cluster_private_test.go index f115b336..2d5b40c1 100644 --- a/test/vault_cluster_private_test.go +++ b/test/vault_cluster_private_test.go @@ -39,14 +39,21 @@ func runVaultPrivateClusterTest(t *testing.T, amiId string, awsRegion string, ss VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), } - deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars) + deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) + }) + + test_structure.RunTestStage(t, "initialize_unseal", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) }) test_structure.RunTestStage(t, "validate", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) - cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) testVaultUsesConsulForDns(t, cluster) }) } diff --git a/test/vault_cluster_public_test.go b/test/vault_cluster_public_test.go index 54f9497b..ec584021 100644 --- a/test/vault_cluster_public_test.go +++ b/test/vault_cluster_public_test.go @@ -46,14 +46,22 @@ func runVaultPublicClusterTest(t *testing.T, amiId string, awsRegion string, ssh VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), } - deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars) + deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) - test_structure.RunTestStage(t, "validate", func() { + test_structure.RunTestStage(t, "initialize_unseal", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) + + test_structure.RunTestStage(t, "validate", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) testVaultViaElb(t, terraformOptions) + testVaultUsesConsulForDns(t, cluster) }) } diff --git a/test/vault_cluster_s3_backend_test.go b/test/vault_cluster_s3_backend_test.go index 16ac3c34..473db2ac 100644 --- a/test/vault_cluster_s3_backend_test.go +++ b/test/vault_cluster_s3_backend_test.go @@ -47,16 +47,18 @@ func runVaultWithS3BackendClusterTest(t *testing.T, amiId string, awsRegion, ssh deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) - test_structure.RunTestStage(t, "validate", func() { + test_structure.RunTestStage(t, "initialize_unseal", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) + + test_structure.RunTestStage(t, "validate", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) - // TODO: temporarily disable DNS check until https://github.com/hashicorp/terraform-aws-consul/issues/155 is - // fixed. See https://github.com/hashicorp/terraform-aws-vault/pull/222 for details. - // - // cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, nil) - // testVaultUsesConsulForDns(t, cluster) + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + testVaultUsesConsulForDns(t, cluster) }) } diff --git a/test/vault_helpers.go b/test/vault_helpers.go index ef041cc1..931126aa 100644 --- a/test/vault_helpers.go +++ b/test/vault_helpers.go @@ -191,6 +191,24 @@ func initializeAndUnsealVaultCluster(t *testing.T, asgNameOutputVar string, sshU return cluster } +// Find the initialized and unsealed Vault cluster, +// exit if cluster is not initialized and unsealed +func getInitializedAndUnsealedVaultCluster(t *testing.T, asgNameOutputVar string, sshUserName string, terraformOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) VaultCluster { + cluster := findVaultClusterNodes(t, asgNameOutputVar, sshUserName, terraformOptions, awsRegion, keyPair) + + establishConnectionToCluster(t, cluster) + isInitializedAndUnsealed, err := isVaultClusterInitializedAndUnsealed(t, cluster) + if err != nil { + t.Logf("Failed to check is vault cluster is already initialized and unsealed: %v", err) + } + // exit if cluster is not initialized and unsealed + if !isInitializedAndUnsealed { + t.Fatalf("Expected to find an initialized and unsealed cluster but it wasn't: %v", cluster) + } + + return cluster +} + // Find the nodes in the given Vault ASG and return them in a VaultCluster struct func findVaultClusterNodes(t *testing.T, asgNameOutputVar string, sshUserName string, terraformOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) VaultCluster { asgName := terraform.Output(t, terraformOptions, asgNameOutputVar) @@ -401,7 +419,7 @@ func boolToTerraformVar(val bool) int { } } -// Check that the Vault node at the given host has the given +// Check that the Vault node at the given host has the given status func assertStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) { description := fmt.Sprintf("Check that the Vault node %s has status %d", host.Hostname, int(expectedStatus)) logger.Logf(t, description) @@ -444,3 +462,45 @@ func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (strin return "", fmt.Errorf("Expected status code %d for host %s, but got %d", int(expectedStatus), host.Hostname, status) } } + +// Check if the given Vault cluster has been initialized and unsealed. +func isVaultClusterInitializedAndUnsealed(t *testing.T, cluster VaultCluster) (bool, error) { + leader, err := hasExpectedStatus(t, cluster.Leader, Leader) + if err != nil { + return false, err + } + standby1, err := hasExpectedStatus(t, cluster.Standby1, Standby) + if err != nil { + return false, err + } + standby2, err := hasExpectedStatus(t, cluster.Standby2, Standby) + if err != nil { + return false, err + } + if leader && standby1 && standby2 { + return true, nil + } else { + return false, nil + } +} + +// Check the status of the given Vault node and ensure it matches the expected status. +// Returns true if the status match, false otherwise. +func hasExpectedStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (bool, error) { + curlCommand := "curl -s -o /dev/null -w '%{http_code}' https://127.0.0.1:8200/v1/sys/health" + logger.Logf(t, "Using curl to check status of Vault server %s: %s", host.Hostname, curlCommand) + + output, err := ssh.CheckSshCommandE(t, host, curlCommand) + if err != nil { + return false, err + } + status, err := strconv.Atoi(output) + if err != nil { + return false, err + } + if status == int(expectedStatus) { + return true, nil + } else { + return false, nil + } +} diff --git a/test/vault_main_test.go b/test/vault_main_test.go index 80d1a88b..220d5580 100644 --- a/test/vault_main_test.go +++ b/test/vault_main_test.go @@ -89,6 +89,7 @@ func TestMainVaultCluster(t *testing.T) { // os.Setenv("SKIP_setup_amis", "true") // os.Setenv("SKIP_deploy", "true") + // os.Setenv("SKIP_initialize_unseal", "true") // os.Setenv("SKIP_validate", "true") // os.Setenv("SKIP_log", "true") // os.Setenv("SKIP_teardown", "true") From 14d4e6d90d25292c05dda1a22965d5934f18bd5b Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Thu, 28 Jan 2021 22:31:23 +0100 Subject: [PATCH 04/14] review: use require.NoError(t, err) --- test/vault_helpers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/vault_helpers.go b/test/vault_helpers.go index 931126aa..d8d1aa37 100644 --- a/test/vault_helpers.go +++ b/test/vault_helpers.go @@ -199,11 +199,11 @@ func getInitializedAndUnsealedVaultCluster(t *testing.T, asgNameOutputVar string establishConnectionToCluster(t, cluster) isInitializedAndUnsealed, err := isVaultClusterInitializedAndUnsealed(t, cluster) if err != nil { - t.Logf("Failed to check is vault cluster is already initialized and unsealed: %v", err) + require.NoError(t, err, "Failed to check is vault cluster is already initialized and unsealed") } // exit if cluster is not initialized and unsealed if !isInitializedAndUnsealed { - t.Fatalf("Expected to find an initialized and unsealed cluster but it wasn't: %v", cluster) + t.Fatalf("Expected to find an initialized and unsealed cluster but it wasn't: [Leader: %s, Standby1:%s , Standby2: %s]", cluster.Leader.Hostname, cluster.Standby1.Hostname, cluster.Standby2.Hostname) } return cluster From e76b2f6fabd44b0c1ff61e760185064511d08bfa Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Thu, 28 Jan 2021 22:32:53 +0100 Subject: [PATCH 05/14] review: configure and use consul_agent_service_registration_address variable for the address of the consul agent to communicate with --- examples/vault-s3-backend/main.tf | 9 +++---- examples/vault-s3-backend/user-data-vault.sh | 2 +- examples/vault-s3-backend/variables.tf | 5 ++++ modules/run-vault/run-vault | 25 +++++++++++++------- test/go.sum | 1 + test/vault_cluster_s3_backend_test.go | 10 ++++---- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/examples/vault-s3-backend/main.tf b/examples/vault-s3-backend/main.tf index c7bac91b..da9e00ea 100644 --- a/examples/vault-s3-backend/main.tf +++ b/examples/vault-s3-backend/main.tf @@ -63,10 +63,11 @@ data "template_file" "user_data_vault_cluster" { template = file("${path.module}/user-data-vault.sh") vars = { - aws_region = data.aws_region.current.name - s3_bucket_name = var.s3_bucket_name - consul_cluster_tag_key = var.consul_cluster_tag_key - consul_cluster_tag_value = var.consul_cluster_name + aws_region = data.aws_region.current.name + s3_bucket_name = var.s3_bucket_name + consul_cluster_tag_key = var.consul_cluster_tag_key + consul_cluster_tag_value = var.consul_cluster_name + consul_agent_service_registration_address = var.consul_agent_service_registration_address } } diff --git a/examples/vault-s3-backend/user-data-vault.sh b/examples/vault-s3-backend/user-data-vault.sh index cfc21ee0..cc1bf530 100644 --- a/examples/vault-s3-backend/user-data-vault.sh +++ b/examples/vault-s3-backend/user-data-vault.sh @@ -16,4 +16,4 @@ readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem" # The variables below are filled in via Terraform interpolation /opt/consul/bin/run-consul --client --cluster-tag-key "${consul_cluster_tag_key}" --cluster-tag-value "${consul_cluster_tag_value}" -/opt/vault/bin/run-vault --tls-cert-file "$VAULT_TLS_CERT_FILE" --tls-key-file "$VAULT_TLS_KEY_FILE" --enable-s3-backend --s3-bucket "${s3_bucket_name}" --s3-bucket-region "${aws_region}" +/opt/vault/bin/run-vault --tls-cert-file "$VAULT_TLS_CERT_FILE" --tls-key-file "$VAULT_TLS_KEY_FILE" --enable-s3-backend --s3-bucket "${s3_bucket_name}" --s3-bucket-region "${aws_region}" --consul-agent-service-registration-address "${consul_agent_service_registration_address}" diff --git a/examples/vault-s3-backend/variables.tf b/examples/vault-s3-backend/variables.tf index f526eaaa..db5ed1bf 100644 --- a/examples/vault-s3-backend/variables.tf +++ b/examples/vault-s3-backend/variables.tf @@ -87,3 +87,8 @@ variable "force_destroy_s3_bucket" { default = false } +variable "consul_agent_service_registration_address" { + description = "Specifies the address of the Consul agent to communicate with. This can be an IP address, DNS record, or unix socket. It is recommended that you communicate with a local Consul agent; do not communicate directly with a server." + type = string + default = "127.0.0.1:8500" +} diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index 0c17fb90..6ae30ee4 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -44,6 +44,7 @@ function print_usage { echo -e " --s3-bucket\tSpecifies the S3 bucket to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-path\tSpecifies the S3 bucket path to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-region\tSpecifies the AWS region where '--s3-bucket' lives. Only used if '--enable-s3-backend' is set." + echo -e " --consul-agent-service-registration-address\tSpecifies the address of the Consul agent to communicate with when using a different storage backend, in this case an S3 backend. Only used if '--enable-s3-backend' is set." echo -e " --enable-dynamo-backend\tIf this flag is set, DynamoDB will be enabled as the backend storage (HA)" echo -e " --dynamo-region\tSpecifies the AWS region where --dynamo-table lives. Only used if '--enable-dynamo-backend is on'" echo -e " --dynamo--table\tSpecifies the DynamoDB table to use for HA Storage. Only used if '--enable-dynamo-backend is on'" @@ -73,7 +74,7 @@ function print_usage { echo echo "Or" echo - echo " run-vault --tls-cert-file /opt/vault/tls/vault.crt.pem --tls-key-file /opt/vault/tls/vault.key.pem --enable-s3-backend --s3-bucket my-vault-bucket --s3-bucket-region us-east-1" + echo " run-vault --tls-cert-file /opt/vault/tls/vault.crt.pem --tls-key-file /opt/vault/tls/vault.key.pem --enable-s3-backend --s3-bucket my-vault-bucket --s3-bucket-region us-east-1 --consul-agent-service-registration-address 127.0.0.1:8500" } function log { @@ -237,13 +238,14 @@ function generate_vault_config { local -r s3_bucket="$9" local -r s3_bucket_path="${10}" local -r s3_bucket_region="${11}" - local -r enable_dynamo_backend="${12}" - local -r dynamo_region="${13}" - local -r dynamo_table="${14}" + local -r consul_agent_service_registration_address="${12}" + local -r enable_dynamo_backend="${13}" + local -r dynamo_region="${14}" + local -r dynamo_table="${15}" local -r enable_auto_unseal="${15}" - local -r auto_unseal_kms_key_id="${16}" - local -r auto_unseal_kms_key_region="${17}" - local -r auto_unseal_endpoint="${18}" + local -r auto_unseal_kms_key_id="${17}" + local -r auto_unseal_kms_key_region="${18}" + local -r auto_unseal_endpoint="${19}" local -r config_path="$config_dir/$VAULT_CONFIG_FILE" local instance_ip_address @@ -302,7 +304,7 @@ EOF dynamodb_storage_type="ha_storage" service_registration=$(cat < Date: Fri, 29 Jan 2021 19:49:25 +0100 Subject: [PATCH 06/14] review: use default value of localhost:8500 for consul-agent-service-registration-address parameter --- examples/vault-s3-backend/main.tf | 1 - examples/vault-s3-backend/user-data-vault.sh | 2 +- examples/vault-s3-backend/variables.tf | 6 ------ modules/run-vault/run-vault | 9 ++++++--- test/vault_cluster_s3_backend_test.go | 10 ++++------ 5 files changed, 11 insertions(+), 17 deletions(-) diff --git a/examples/vault-s3-backend/main.tf b/examples/vault-s3-backend/main.tf index da9e00ea..d6c264ae 100644 --- a/examples/vault-s3-backend/main.tf +++ b/examples/vault-s3-backend/main.tf @@ -67,7 +67,6 @@ data "template_file" "user_data_vault_cluster" { s3_bucket_name = var.s3_bucket_name consul_cluster_tag_key = var.consul_cluster_tag_key consul_cluster_tag_value = var.consul_cluster_name - consul_agent_service_registration_address = var.consul_agent_service_registration_address } } diff --git a/examples/vault-s3-backend/user-data-vault.sh b/examples/vault-s3-backend/user-data-vault.sh index cc1bf530..cfc21ee0 100644 --- a/examples/vault-s3-backend/user-data-vault.sh +++ b/examples/vault-s3-backend/user-data-vault.sh @@ -16,4 +16,4 @@ readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem" # The variables below are filled in via Terraform interpolation /opt/consul/bin/run-consul --client --cluster-tag-key "${consul_cluster_tag_key}" --cluster-tag-value "${consul_cluster_tag_value}" -/opt/vault/bin/run-vault --tls-cert-file "$VAULT_TLS_CERT_FILE" --tls-key-file "$VAULT_TLS_KEY_FILE" --enable-s3-backend --s3-bucket "${s3_bucket_name}" --s3-bucket-region "${aws_region}" --consul-agent-service-registration-address "${consul_agent_service_registration_address}" +/opt/vault/bin/run-vault --tls-cert-file "$VAULT_TLS_CERT_FILE" --tls-key-file "$VAULT_TLS_KEY_FILE" --enable-s3-backend --s3-bucket "${s3_bucket_name}" --s3-bucket-region "${aws_region}" diff --git a/examples/vault-s3-backend/variables.tf b/examples/vault-s3-backend/variables.tf index db5ed1bf..d73a2590 100644 --- a/examples/vault-s3-backend/variables.tf +++ b/examples/vault-s3-backend/variables.tf @@ -86,9 +86,3 @@ variable "force_destroy_s3_bucket" { type = bool default = false } - -variable "consul_agent_service_registration_address" { - description = "Specifies the address of the Consul agent to communicate with. This can be an IP address, DNS record, or unix socket. It is recommended that you communicate with a local Consul agent; do not communicate directly with a server." - type = string - default = "127.0.0.1:8500" -} diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index 6ae30ee4..362cf349 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -14,6 +14,8 @@ readonly DEFAULT_AGENT_AUTH_MOUNT_PATH="auth/aws" readonly DEFAULT_PORT=8200 readonly DEFAULT_LOG_LEVEL="info" +readonly DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS="localhost:8500" + readonly EC2_INSTANCE_METADATA_URL="http://169.254.169.254/latest/meta-data" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -44,7 +46,7 @@ function print_usage { echo -e " --s3-bucket\tSpecifies the S3 bucket to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-path\tSpecifies the S3 bucket path to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-region\tSpecifies the AWS region where '--s3-bucket' lives. Only used if '--enable-s3-backend' is set." - echo -e " --consul-agent-service-registration-address\tSpecifies the address of the Consul agent to communicate with when using a different storage backend, in this case an S3 backend. Only used if '--enable-s3-backend' is set." + echo -e " --consul-agent-service-registration-address\tSpecifies the address of the Consul agent to communicate with when using a different storage backend, in this case an S3 backend. Only used if '--enable-s3-backend' is set. Default is $DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS." echo -e " --enable-dynamo-backend\tIf this flag is set, DynamoDB will be enabled as the backend storage (HA)" echo -e " --dynamo-region\tSpecifies the AWS region where --dynamo-table lives. Only used if '--enable-dynamo-backend is on'" echo -e " --dynamo--table\tSpecifies the DynamoDB table to use for HA Storage. Only used if '--enable-dynamo-backend is on'" @@ -74,7 +76,7 @@ function print_usage { echo echo "Or" echo - echo " run-vault --tls-cert-file /opt/vault/tls/vault.crt.pem --tls-key-file /opt/vault/tls/vault.key.pem --enable-s3-backend --s3-bucket my-vault-bucket --s3-bucket-region us-east-1 --consul-agent-service-registration-address 127.0.0.1:8500" + echo " run-vault --tls-cert-file /opt/vault/tls/vault.crt.pem --tls-key-file /opt/vault/tls/vault.key.pem --enable-s3-backend --s3-bucket my-vault-bucket --s3-bucket-region us-east-1" } function log { @@ -463,7 +465,7 @@ function run { local s3_bucket="" local s3_bucket_path="" local s3_bucket_region="" - local consul_agent_service_registration_address="" + local consul_agent_service_registration_address="$DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS" local enable_dynamo_backend="false" local dynamo_region="" local dynamo_table="" @@ -563,6 +565,7 @@ function run { shift ;; --consul-agent-service-registration-address) + assert_not_empty "$key" "$2" consul_agent_service_registration_address="$2" shift ;; diff --git a/test/vault_cluster_s3_backend_test.go b/test/vault_cluster_s3_backend_test.go index c6b4d350..473db2ac 100644 --- a/test/vault_cluster_s3_backend_test.go +++ b/test/vault_cluster_s3_backend_test.go @@ -12,7 +12,6 @@ const VAULT_CLUSTER_S3_BACKEND_PATH = "examples/vault-s3-backend" const VAR_S3_BUCKET_NAME = "s3_bucket_name" const VAR_FORCE_DESTROY_S3_BUCKET = "force_destroy_s3_bucket" -const VAR_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS = "consul_agent_service_registration_address" // Test the Vault with S3 Backend example by: // @@ -40,11 +39,10 @@ func runVaultWithS3BackendClusterTest(t *testing.T, amiId string, awsRegion, ssh test_structure.RunTestStage(t, "deploy", func() { uniqueId := random.UniqueId() terraformVars := map[string]interface{}{ - VAR_S3_BUCKET_NAME: s3BucketName(uniqueId), - VAR_FORCE_DESTROY_S3_BUCKET: true, - VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), - VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), - VAR_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS: "127.0.0.1:8500", + VAR_S3_BUCKET_NAME: s3BucketName(uniqueId), + VAR_FORCE_DESTROY_S3_BUCKET: true, + VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), + VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), } deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) From 9781c220d5f3e56836976aa47486d2c67c5b892f Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Fri, 29 Jan 2021 22:54:22 +0100 Subject: [PATCH 07/14] pre-commit hook --- examples/vault-s3-backend/main.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/vault-s3-backend/main.tf b/examples/vault-s3-backend/main.tf index d6c264ae..c7bac91b 100644 --- a/examples/vault-s3-backend/main.tf +++ b/examples/vault-s3-backend/main.tf @@ -63,10 +63,10 @@ data "template_file" "user_data_vault_cluster" { template = file("${path.module}/user-data-vault.sh") vars = { - aws_region = data.aws_region.current.name - s3_bucket_name = var.s3_bucket_name - consul_cluster_tag_key = var.consul_cluster_tag_key - consul_cluster_tag_value = var.consul_cluster_name + aws_region = data.aws_region.current.name + s3_bucket_name = var.s3_bucket_name + consul_cluster_tag_key = var.consul_cluster_tag_key + consul_cluster_tag_value = var.consul_cluster_name } } From 971e3b34c282577d6ddb565416530ae99b1f810d Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Fri, 29 Jan 2021 22:58:28 +0100 Subject: [PATCH 08/14] update pre-commit config to the latest repos' versions --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81355c81..5db53b5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/gruntwork-io/pre-commit - rev: v0.1.10 + rev: v0.1.12 hooks: - id: terraform-fmt - id: gofmt \ No newline at end of file From 9eb6c58b1b9113e840be869bb25b394702529d3f Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Tue, 9 Feb 2021 22:17:52 +0100 Subject: [PATCH 09/14] Using same format '${variable_name}' for all variables --- modules/run-vault/run-vault | 482 ++++++++++++++++++------------------ 1 file changed, 241 insertions(+), 241 deletions(-) diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index 362cf349..28ab7c34 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -19,7 +19,7 @@ readonly DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS="localhost:8500" readonly EC2_INSTANCE_METADATA_URL="http://169.254.169.254/latest/meta-data" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly SCRIPT_NAME="$(basename "$0")" +readonly SCRIPT_NAME="$(basename "${0}")" function print_usage { echo @@ -31,13 +31,13 @@ function print_usage { echo echo -e " --tls-cert-file\tSpecifies the path to the certificate for TLS. Required. To use a CA certificate, concatenate the primary certificate and the CA certificate together." echo -e " --tls-key-file\tSpecifies the path to the private key for the certificate. Required." - echo -e " --port\t\tThe port for Vault to listen on. Optional. Default is $DEFAULT_PORT." + echo -e " --port\t\tThe port for Vault to listen on. Optional. Default is ${DEFAULT_PORT}." echo -e " --cluster-port\tThe port for Vault to listen on for server-to-server requests. Optional. Default is --port + 1." - echo -e " --api-addr\t\tThe full address to use for Client Redirection when running Vault in HA mode. Defaults to \"https://[instance_ip]:$DEFAULT_PORT\". Optional." + echo -e " --api-addr\t\tThe full address to use for Client Redirection when running Vault in HA mode. Defaults to \"https://[instance_ip]:${DEFAULT_PORT}\". Optional." echo -e " --config-dir\t\tThe path to the Vault config folder. Optional. Default is the absolute path of '../config', relative to this script." echo -e " --bin-dir\t\tThe path to the folder with Vault binary. Optional. Default is the absolute path of the parent folder of this script." echo -e " --data-dir\t\tThe path to the Vault data folder. Optional. Default is the absolute path of '../data', relative to this script." - echo -e " --log-level\t\tThe log verbosity to use with Vault. Optional. Default is $DEFAULT_LOG_LEVEL." + echo -e " --log-level\t\tThe log verbosity to use with Vault. Optional. Default is ${DEFAULT_LOG_LEVEL}." echo -e " --systemd-stdout\t\tThe StandardOutput option of the systemd unit. Optional. If not configured, uses systemd's default (journal)." echo -e " --systemd-stderr\t\tThe StandardError option of the systemd unit. Optional. If not configured, uses systemd's default (inherit)." echo -e " --user\t\tThe user to run Vault as. Optional. Default is to use the owner of --config-dir." @@ -46,7 +46,7 @@ function print_usage { echo -e " --s3-bucket\tSpecifies the S3 bucket to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-path\tSpecifies the S3 bucket path to use to store Vault data. Only used if '--enable-s3-backend' is set." echo -e " --s3-bucket-region\tSpecifies the AWS region where '--s3-bucket' lives. Only used if '--enable-s3-backend' is set." - echo -e " --consul-agent-service-registration-address\tSpecifies the address of the Consul agent to communicate with when using a different storage backend, in this case an S3 backend. Only used if '--enable-s3-backend' is set. Default is $DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS." + echo -e " --consul-agent-service-registration-address\tSpecifies the address of the Consul agent to communicate with when using a different storage backend, in this case an S3 backend. Only used if '--enable-s3-backend' is set. Default is ${DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS}." echo -e " --enable-dynamo-backend\tIf this flag is set, DynamoDB will be enabled as the backend storage (HA)" echo -e " --dynamo-region\tSpecifies the AWS region where --dynamo-table lives. Only used if '--enable-dynamo-backend is on'" echo -e " --dynamo--table\tSpecifies the DynamoDB table to use for HA Storage. Only used if '--enable-dynamo-backend is on'" @@ -54,12 +54,12 @@ function print_usage { echo "Options for Vault Agent:" echo echo -e " --agent\t\t\tIf set, run in Vault Agent mode. If not set, run as a regular Vault server. Optional." - echo -e " --agent-vault-address\t\tThe hostname or IP address of the Vault server to connect to. Optional. Default is $DEFAULT_AGENT_VAULT_ADDRESS" - echo -e " --agent-vault-port\t\tThe port of the Vault server to connect to. Optional. Default is $DEFAULT_PORT" + echo -e " --agent-vault-address\t\tThe hostname or IP address of the Vault server to connect to. Optional. Default is ${DEFAULT_AGENT_VAULT_ADDRESS}" + echo -e " --agent-vault-port\t\tThe port of the Vault server to connect to. Optional. Default is ${DEFAULT_PORT}" echo -e " --agent-ca-cert-file\t\tSpecifies the path to a CA certificate to verify the Vault server's TLS certificate. Optional." echo -e " --agent-client-cert-file\tSpecifies the path to a certificate to use for TLS authentication to the Vault server. Optional." echo -e " --agent-client-key-file\tSpecifies the path to the private key for the client certificate used for TLS authentication to the Vault server. Optional." - echo -e " --agent-auth-mount-path\tThe Vault mount path to the auth method used for auto-auth. Optional. Defaults to $DEFAULT_AGENT_AUTH_MOUNT_PATH" + echo -e " --agent-auth-mount-path\tThe Vault mount path to the auth method used for auto-auth. Optional. Defaults to ${DEFAULT_AGENT_AUTH_MOUNT_PATH}" echo -e " --agent-auth-type\t\tThe Vault AWS auth type to use for auto-auth. Required. Must be either iam or ec2" echo -e " --agent-auth-role\t\tThe Vault role to authenticate against. Required." echo @@ -80,48 +80,48 @@ function print_usage { } function log { - local -r level="$1" - local -r message="$2" + local -r level="${1}" + local -r message="${2}" local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S") - >&2 echo -e "${timestamp} [${level}] [$SCRIPT_NAME] ${message}" + >&2 echo -e "${timestamp} [${level}] [${SCRIPT_NAME}] ${message}" } function log_info { - local -r message="$1" - log "INFO" "$message" + local -r message="${1}" + log "INFO" "${message}" } function log_warn { - local -r message="$1" - log "WARN" "$message" + local -r message="${1}" + log "WARN" "${message}" } function log_error { - local -r message="$1" - log "ERROR" "$message" + local -r message="${1}" + log "ERROR" "${message}" } # Based on code from: http://stackoverflow.com/a/16623897/483528 function strip_prefix { - local -r str="$1" - local -r prefix="$2" + local -r str="${1}" + local -r prefix="${2}" echo "${str#$prefix}" } function assert_not_empty { - local -r arg_name="$1" - local -r arg_value="$2" + local -r arg_name="${1}" + local -r arg_value="${2}" - if [[ -z "$arg_value" ]]; then - log_error "The value for '$arg_name' cannot be empty" + if [[ -z "${arg_value}" ]]; then + log_error "The value for '${arg_name}' cannot be empty" print_usage exit 1 fi } function lookup_path_in_instance_metadata { - local -r path="$1" - curl --silent --location "$EC2_INSTANCE_METADATA_URL/$path/" + local -r path="${1}" + curl --silent --location "${EC2_INSTANCE_METADATA_URL}/${path}/" } function get_instance_ip_address { @@ -129,10 +129,10 @@ function get_instance_ip_address { } function assert_is_installed { - local -r name="$1" + local -r name="${1}" if [[ ! $(command -v ${name}) ]]; then - log_error "The binary '$name' is required by this script but is not installed or in the system's PATH." + log_error "The binary '${name}' is required by this script but is not installed or in the system's PATH." exit 1 fi } @@ -145,60 +145,60 @@ function get_vault_version { } function vault_version_at_least { - local -r config_path=$1 - local -r ui_config=$2 + local -r config_path="${1}" + local -r ui_config="${2}" VAULT_VERSION=$(get_vault_version) #This if statement will echo the current vault version and the minimum version required for ui support. #It then strips out the comments, sorts the two values, and chooses the top (least) one. - if [[ $(echo "$VAULT_VERSION 0.10.0" | tr " " "\n" |sort --version-sort| head -n 1) = 0.10.0 ]] + if [[ $(echo "${VAULT_VERSION} 0.10.0" | tr " " "\n" |sort --version-sort| head -n 1) = 0.10.0 ]] then - echo -e "$ui_config" >> "$config_path" + echo -e "${ui_config}" >> "${config_path}" else log_info "Vault 0.10.0 or greater is required for UI support." fi } function generate_vault_agent_config { - local -r config_dir="$1" - local -r data_dir="$2" - local -r vault_address="$3" - local -r vault_port="$4" - local -r ca_cert_file="$5" - local -r client_cert_file="$6" - local -r client_key_file="$7" - local -r auth_mount_path="$8" - local -r auth_type="$9" + local -r config_dir="${1}" + local -r data_dir="${2}" + local -r vault_address="${3}" + local -r vault_port="${4}" + local -r ca_cert_file="${5}" + local -r client_cert_file="${6}" + local -r client_key_file="${7}" + local -r auth_mount_path="${8}" + local -r auth_type="${9}" local -r auth_role="${10}" - local -r config_path="$config_dir/$VAULT_CONFIG_FILE" + local -r config_path="${config_dir}/${VAULT_CONFIG_FILE}" - log_info "Creating default Vault Agent config file in $config_path" + log_info "Creating default Vault Agent config file in ${config_path}" - local -r pid_config="pid_file = \"$data_dir/$VAULT_PID_FILE\"" + local -r pid_config="pid_file = \"${data_dir}/$VAULT_PID_FILE\"" local ca_cert_config="" - if [[ ! -z $ca_cert_file ]]; then + if [[ ! -z ${ca_cert_file} ]]; then ca_cert_config=$(cat < "$config_path" - echo -e "$vault_config" >> "$config_path" - echo -e "$auto_auth_config" >> "$config_path" + echo -e "${pid_config}" > "${config_path}" + echo -e "${vault_config}" >> "${config_path}" + echo -e "${auto_auth_config}" >> "${config_path}" - chown "$user:$user" "$config_path" + chown "${user}:${user}" "${config_path}" } function generate_vault_config { - local -r tls_cert_file="$1" - local -r tls_key_file="$2" - local -r port="$3" - local -r cluster_port="$4" - local -r api_addr="$5" - local -r config_dir="$6" - local -r user="$7" - local -r enable_s3_backend="$8" - local -r s3_bucket="$9" + local -r tls_cert_file="${1}" + local -r tls_key_file="${2}" + local -r port="${3}" + local -r cluster_port="${4}" + local -r api_addr="${5}" + local -r config_dir="${6}" + local -r user="${7}" + local -r enable_s3_backend="${8}" + local -r s3_bucket="${9}" local -r s3_bucket_path="${10}" local -r s3_bucket_region="${11}" local -r consul_agent_service_registration_address="${12}" local -r enable_dynamo_backend="${13}" local -r dynamo_region="${14}" local -r dynamo_table="${15}" - local -r enable_auto_unseal="${15}" + local -r enable_auto_unseal="${16}" local -r auto_unseal_kms_key_id="${17}" local -r auto_unseal_kms_key_region="${18}" local -r auto_unseal_endpoint="${19}" - local -r config_path="$config_dir/$VAULT_CONFIG_FILE" + local -r config_path="${config_dir}/${VAULT_CONFIG_FILE}" local instance_ip_address instance_ip_address=$(get_instance_ip_address) local auto_unseal_config="" - if [[ "$enable_auto_unseal" == "true" ]]; then + if [[ "${enable_auto_unseal}" == "true" ]]; then local endpoint="" - if [[ -n "$auto_unseal_endpoint" ]]; then - endpoint="endpoint = \"$auto_unseal_endpoint\"" + if [[ -n "${auto_unseal_endpoint}" ]]; then + endpoint="endpoint = \"${auto_unseal_endpoint}\"" fi auto_unseal_config=$(cat <> "$config_path" - echo -e "$listener_config" >> "$config_path" - echo -e "$s3_config" >> "$config_path" - echo -e "$vault_storage_backend" >> "$config_path" - echo -e "$service_registration" >> "$config_path" + echo -e "${auto_unseal_config}" >> "${config_path}" + echo -e "${listener_config}" >> "${config_path}" + echo -e "${s3_config}" >> "${config_path}" + echo -e "${vault_storage_backend}" >> "${config_path}" + echo -e "${service_registration}" >> "${config_path}" - chown "$user:$user" "$config_path" + chown "${user}:${user}" "${config_path}" } function generate_systemd_config { - local -r systemd_config_path="$1" - local -r vault_config_dir="$2" - local -r vault_bin_dir="$3" - local -r vault_log_level="$4" - local -r vault_systemd_stdout="$5" - local -r vault_systemd_stderr="$6" - local -r vault_user="$7" - local -r is_vault_agent="$8" - local -r config_path="$config_dir/$VAULT_CONFIG_FILE" + local -r systemd_config_path="${1}" + local -r vault_config_dir="${2}" + local -r vault_bin_dir="${3}" + local -r vault_log_level="${4}" + local -r vault_systemd_stdout="${5}" + local -r vault_systemd_stderr="${6}" + local -r vault_user="${7}" + local -r is_vault_agent="${8}" + local -r config_path="${config_dir}/${VAULT_CONFIG_FILE}" local vault_description="HashiCorp Vault - A tool for managing secrets" local vault_command="server" @@ -367,18 +367,18 @@ function generate_systemd_config { if [[ "$is_vault_agent" == "true" ]]; then vault_command="agent" vault_description="HashiCorp Vault Agent" - vault_config_file_or_dir="$config_path" + vault_config_file_or_dir="${config_path}" fi - log_info "Creating systemd config file to run Vault in $systemd_config_path" + log_info "Creating systemd config file to run Vault in ${systemd_config_path}" local -r unit_config=$(cat < "$systemd_config_path" - echo -e "$service_config" >> "$systemd_config_path" - echo -e "$log_config" >> "$systemd_config_path" - echo -e "$install_config" >> "$systemd_config_path" + echo -e "${unit_config}" > "${systemd_config_path}" + echo -e "${service_config}" >> "${systemd_config_path}" + echo -e "${log_config}" >> "${systemd_config_path}" + echo -e "${install_config}" >> "${systemd_config_path}" } function start_vault { @@ -443,20 +443,20 @@ function start_vault { # Based on: http://unix.stackexchange.com/a/7732/215969 function get_owner_of_path { - local -r path="$1" - ls -ld "$path" | awk '{print $3}' + local -r path="${1}" + ls -ld "${path}" | awk '{print $3}' } function run { local tls_cert_file="" local tls_key_file="" - local port="$DEFAULT_PORT" + local port="${DEFAULT_PORT}" local cluster_port="" local api_addr="" local config_dir="" local bin_dir="" local data_dir="" - local log_level="$DEFAULT_LOG_LEVEL" + local log_level="${DEFAULT_LOG_LEVEL}" local systemd_stdout="" local systemd_stderr="" local user="" @@ -465,17 +465,17 @@ function run { local s3_bucket="" local s3_bucket_path="" local s3_bucket_region="" - local consul_agent_service_registration_address="$DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS" + local consul_agent_service_registration_address="${DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS}" local enable_dynamo_backend="false" local dynamo_region="" local dynamo_table="" local agent="false" - local agent_vault_address="$DEFAULT_AGENT_VAULT_ADDRESS" - local agent_vault_port="$DEFAULT_PORT" + local agent_vault_address="${DEFAULT_AGENT_VAULT_ADDRESS}" + local agent_vault_port="${DEFAULT_PORT}" local agent_ca_cert_file="" local agent_client_cert_file="" local agent_client_key_file="" - local agent_auth_mount_path="$DEFAULT_AGENT_AUTH_MOUNT_PATH" + local agent_auth_mount_path="${DEFAULT_AGENT_AUTH_MOUNT_PATH}" local agent_auth_type="" local agent_auth_role="" local enable_auto_unseal="false" @@ -485,65 +485,65 @@ function run { local all_args=() while [[ $# > 0 ]]; do - local key="$1" + local key="${1}" - case "$key" in + case "${key}" in --tls-cert-file) - tls_cert_file="$2" + tls_cert_file="${2}" shift ;; --tls-key-file) - tls_key_file="$2" + tls_key_file="${2}" shift ;; --port) - assert_not_empty "$key" "$2" - port="$2" + assert_not_empty "${key}" "${2}" + port="${2}" shift ;; --cluster-port) - assert_not_empty "$key" "$2" - cluster_port="$2" + assert_not_empty "${key}" "${2}" + cluster_port="${2}" shift ;; --config-dir) - assert_not_empty "$key" "$2" - config_dir="$2" + assert_not_empty "${key}" "${2}" + config_dir="${2}" shift ;; --api-addr) - assert_not_empty "$key" "$2" - api_addr="$2" + assert_not_empty "${key}" "${2}" + api_addr="${2}" shift ;; --bin-dir) - assert_not_empty "$key" "$2" - bin_dir="$2" + assert_not_empty "${key}" "${2}" + bin_dir="${2}" shift ;; --data-dir) - assert_not_empty "$key" "$2" - data_dir="$2" + assert_not_empty "${key}" "${2}" + data_dir="${2}" shift ;; --log-level) - assert_not_empty "$key" "$2" - log_level="$2" + assert_not_empty "${key}" "${2}" + log_level="${2}" shift ;; --systemd-stdout) - assert_not_empty "$key" "$2" - systemd_stdout="$2" + assert_not_empty "${key}" "${2}" + systemd_stdout="${2}" shift ;; --systemd-stderr) - assert_not_empty "$key" "$2" - systemd_stderr="$2" + assert_not_empty "${key}" "${2}" + systemd_stderr="${2}" shift ;; --user) - assert_not_empty "$key" "$2" - user="$2" + assert_not_empty "${key}" "${2}" + user="${2}" shift ;; --skip-vault-config) @@ -553,87 +553,87 @@ function run { enable_s3_backend="true" ;; --s3-bucket) - s3_bucket="$2" + s3_bucket="${2}" shift ;; --s3-bucket-path) - s3_bucket_path="$2" + s3_bucket_path="${2}" shift ;; --s3-bucket-region) - s3_bucket_region="$2" + s3_bucket_region="${2}" shift ;; --consul-agent-service-registration-address) - assert_not_empty "$key" "$2" - consul_agent_service_registration_address="$2" + assert_not_empty "${key}" "${2}" + consul_agent_service_registration_address="${2}" shift ;; --enable-dynamo-backend) enable_dynamo_backend="true" ;; --dynamo-region) - dynamo_region="$2" + dynamo_region="${2}" shift ;; --dynamo-table) - dynamo_table="$2" + dynamo_table="${2}" shift ;; --agent) agent="true" ;; --agent-vault-address) - assert_not_empty "$key" "$2" - agent_vault_address="$2" + assert_not_empty "${key}" "${2}" + agent_vault_address="${2}" shift ;; --agent-vault-port) - assert_not_empty "$key" "$2" - agent_vault_port="$2" + assert_not_empty "${key}" "${2}" + agent_vault_port="${2}" shift ;; --agent-ca-cert-file) - assert_not_empty "$key" "$2" - agent_ca_cert_file="$2" + assert_not_empty "${key}" "${2}" + agent_ca_cert_file="${2}" shift ;; --agent-client-cert-file) - assert_not_empty "$key" "$2" - agent_client_cert_file="$2" + assert_not_empty "${key}" "${2}" + agent_client_cert_file="${2}" shift ;; --agent-client-key-file) - assert_not_empty "$key" "$2" - agent_client_key_file="$2" + assert_not_empty "${key}" "${2}" + agent_client_key_file="${2}" shift ;; --agent-auth-mount-path) - assert_not_empty "$key" "$2" - agent_auth_mount_path="$2" + assert_not_empty "${key}" "${2}" + agent_auth_mount_path="${2}" shift ;; --agent-auth-type) - agent_auth_type="$2" + agent_auth_type="${2}" shift ;; --agent-auth-role) - agent_auth_role="$2" + agent_auth_role="${2}" shift ;; --enable-auto-unseal) enable_auto_unseal="true" ;; --auto-unseal-kms-key-id) - auto_unseal_kms_key_id="$2" + auto_unseal_kms_key_id="${2}" shift ;; --auto-unseal-kms-key-region) - auto_unseal_kms_key_region="$2" + auto_unseal_kms_key_region="${2}" shift ;; --auto-unseal-endpoint) - auto_unseal_endpoint="$2" + auto_unseal_endpoint="${2}" shift ;; --help) @@ -641,7 +641,7 @@ function run { exit ;; *) - log_error "Unrecognized argument: $key" + log_error "Unrecognized argument: ${key}" print_usage exit 1 ;; @@ -651,23 +651,23 @@ function run { done # Required flags - if [[ "$agent" == "true" ]]; then - assert_not_empty "--agent-auth-type" "$agent_auth_type" - assert_not_empty "--agent-auth-role" "$agent_auth_role" + if [[ "${agent}" == "true" ]]; then + assert_not_empty "--agent-auth-type" "${agent_auth_type}" + assert_not_empty "--agent-auth-role" "${agent_auth_role}" else - assert_not_empty "--tls-cert-file" "$tls_cert_file" - assert_not_empty "--tls-key-file" "$tls_key_file" + assert_not_empty "--tls-cert-file" "${tls_cert_file}" + assert_not_empty "--tls-key-file" "${tls_key_file}" - if [[ "$enable_s3_backend" == "true" ]]; then - assert_not_empty "--s3-bucket" "$s3_bucket" - assert_not_empty "--s3-bucket-region" "$s3_bucket_region" - assert_not_empty "--consul-agent-service-registration-address" "$consul_agent_service_registration_address" + if [[ "${enable_s3_backend}" == "true" ]]; then + assert_not_empty "--s3-bucket" "${s3_bucket}" + assert_not_empty "--s3-bucket-region" "${s3_bucket_region}" + assert_not_empty "--consul-agent-service-registration-address" "${consul_agent_service_registration_address}" fi fi - if [[ "$enable_dynamo_backend" == "true" ]]; then - assert_not_empty "--dynamo-table" "$dynamo_table" - assert_not_empty "--dynamo-region" "$dynamo_region" + if [[ "${enable_dynamo_backend}" == "true" ]]; then + assert_not_empty "--dynamo-table" "${dynamo_table}" + assert_not_empty "--dynamo-region" "${dynamo_region}" fi assert_is_installed "systemctl" @@ -675,80 +675,80 @@ function run { assert_is_installed "curl" assert_is_installed "jq" - if [[ -z "$config_dir" ]]; then - config_dir=$(cd "$SCRIPT_DIR/../config" && pwd) + if [[ -z "${config_dir}" ]]; then + config_dir=$(cd "${SCRIPT_DIR}/../config" && pwd) fi - # If $systemd_stdout and/or $systemd_stderr are empty, we leave them empty so that generate_systemd_config will use systemd's defaults (journal and inherit, respectively) + # If ${systemd_stdout} and/or ${systemd_stderr} are empty, we leave them empty so that generate_systemd_config will use systemd's defaults (journal and inherit, respectively) - if [[ -z "$bin_dir" ]]; then - bin_dir=$(cd "$SCRIPT_DIR/../bin" && pwd) + if [[ -z "${bin_dir}" ]]; then + bin_dir=$(cd "${SCRIPT_DIR}/../bin" && pwd) fi - if [[ -z "$data_dir" ]]; then - data_dir=$(cd "$SCRIPT_DIR/../data" && pwd) + if [[ -z "${data_dir}" ]]; then + data_dir=$(cd "${SCRIPT_DIR}/../data" && pwd) fi - if [[ -z "$user" ]]; then - user=$(get_owner_of_path "$config_dir") + if [[ -z "${user}" ]]; then + user=$(get_owner_of_path "${config_dir}") fi - if [[ -z "$cluster_port" ]]; then - cluster_port=$(( $port + 1 )) + if [[ -z "${cluster_port}" ]]; then + cluster_port=$(( ${port} + 1 )) fi - if [[ -z "$api_addr" ]]; then + if [[ -z "${api_addr}" ]]; then api_addr="https://$(get_instance_ip_address):${port}" fi - if [[ "$agent" == "false" ]] && [[ "$enable_auto_unseal" == "true" ]]; then + if [[ "${agent}" == "false" ]] && [[ "${enable_auto_unseal}" == "true" ]]; then log_info "The --enable-auto-unseal flag is set to true" - assert_not_empty "--auto-unseal-kms-key-id" "$auto_unseal_kms_key_id" - assert_not_empty "--auto-unseal-kms-key-region" "$auto_unseal_kms_key_region" + assert_not_empty "--auto-unseal-kms-key-id" "${auto_unseal_kms_key_id}" + assert_not_empty "--auto-unseal-kms-key-region" "${auto_unseal_kms_key_region}" fi - if [[ "$skip_vault_config" == "true" ]]; then + if [[ "${skip_vault_config}" == "true" ]]; then log_info "The --skip-vault-config flag is set, so will not generate a default Vault config file." else - if [[ "$agent" == "true" ]]; then + if [[ "${agent}" == "true" ]]; then log_info "Running as Vault agent (--agent flag is set)" generate_vault_agent_config \ - "$config_dir" \ - "$data_dir" \ - "$agent_vault_address" \ - "$agent_vault_port" \ - "$agent_ca_cert_file" \ - "$agent_client_cert_file" \ - "$agent_client_key_file" \ - "$agent_auth_mount_path" \ - "$agent_auth_type" \ - "$agent_auth_role" + "${config_dir}" \ + "${data_dir}" \ + "${agent_vault_address}" \ + "${agent_vault_port}" \ + "${agent_ca_cert_file}" \ + "${agent_client_cert_file}" \ + "${agent_client_key_file}" \ + "${agent_auth_mount_path}" \ + "${agent_auth_type}" \ + "${agent_auth_role}" else log_info "Running as Vault server" generate_vault_config \ - "$tls_cert_file" \ - "$tls_key_file" \ - "$port" \ - "$cluster_port" \ - "$api_addr" \ - "$config_dir" \ - "$user" \ - "$enable_s3_backend" \ - "$s3_bucket" \ - "$s3_bucket_path" \ - "$s3_bucket_region" \ - "$consul_agent_service_registration_address" \ - "$enable_dynamo_backend" \ - "$dynamo_region" \ - "$dynamo_table" \ - "$enable_auto_unseal" \ - "$auto_unseal_kms_key_id" \ - "$auto_unseal_kms_key_region" \ - "$auto_unseal_endpoint" + "${tls_cert_file}" \ + "${tls_key_file}" \ + "${port}" \ + "${cluster_port}" \ + "${api_addr}" \ + "${config_dir}" \ + "${user}" \ + "${enable_s3_backend}" \ + "${s3_bucket}" \ + "${s3_bucket_path}" \ + "${s3_bucket_region}" \ + "${consul_agent_service_registration_address}" \ + "${enable_dynamo_backend}" \ + "${dynamo_region}" \ + "${dynamo_table}" \ + "${enable_auto_unseal}" \ + "${auto_unseal_kms_key_id}" \ + "${auto_unseal_kms_key_region}" \ + "${auto_unseal_endpoint}" fi fi - generate_systemd_config "$SYSTEMD_CONFIG_PATH" "$config_dir" "$bin_dir" "$log_level" "$systemd_stdout" "$systemd_stderr" "$user" "$agent" + generate_systemd_config "${SYSTEMD_CONFIG_PATH}" "${config_dir}" "${bin_dir}" "${log_level}" "${systemd_stdout}" "${systemd_stderr}" "${user}" "${agent}" start_vault } From 279a9b31526dd66fc7b73de312768cd1b33a6cad Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Tue, 9 Feb 2021 23:02:49 +0100 Subject: [PATCH 10/14] Updating vault-dynamodb-backend example: enable S3 storage backend --- examples/vault-dynamodb-backend/main.tf | 18 +++++++++++++++++- .../vault-dynamodb-backend/user-data-vault.sh | 5 ++++- examples/vault-dynamodb-backend/variables.tf | 12 ++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/examples/vault-dynamodb-backend/main.tf b/examples/vault-dynamodb-backend/main.tf index 169027c5..3b402a53 100644 --- a/examples/vault-dynamodb-backend/main.tf +++ b/examples/vault-dynamodb-backend/main.tf @@ -9,7 +9,7 @@ terraform { } # --------------------------------------------------------------------------------------------------------------------- -# DEPLOY THE VAULT SERVER CLUSTER +# DEPLOY THE DYNAMODB STORAGE BACKEND # --------------------------------------------------------------------------------------------------------------------- module "backend" { @@ -19,6 +19,10 @@ module "backend" { write_capacity = var.dynamo_write_capacity } +# --------------------------------------------------------------------------------------------------------------------- +# DEPLOY THE VAULT SERVER CLUSTER +# --------------------------------------------------------------------------------------------------------------------- + module "vault_cluster" { # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you # to a specific version of the modules, such as the following example: @@ -32,6 +36,11 @@ module "vault_cluster" { ami_id = var.ami_id user_data = data.template_file.user_data_vault_cluster.rendered + # Enable S3 storage backend + enable_s3_backend = true + s3_bucket_name = var.s3_bucket_name + force_destroy_s3_bucket = var.force_destroy_s3_bucket + vpc_id = data.aws_vpc.default.id subnet_ids = data.aws_subnet_ids.default.ids @@ -44,16 +53,23 @@ module "vault_cluster" { allowed_inbound_security_group_count = 0 ssh_key_name = var.ssh_key_name + # Enable DynamoDB high availability storage backend enable_dynamo_backend = true dynamo_table_name = var.dynamo_table_name } +# --------------------------------------------------------------------------------------------------------------------- +# THE USER DATA SCRIPT THAT WILL RUN ON EACH VAULT SERVER WHEN IT'S BOOTING +# This script will configure and start Vault +# --------------------------------------------------------------------------------------------------------------------- + data "template_file" "user_data_vault_cluster" { template = file("${path.module}/user-data-vault.sh") vars = { aws_region = data.aws_region.current.name dynamo_table_name = var.dynamo_table_name + s3_bucket_name = var.s3_bucket_name } } diff --git a/examples/vault-dynamodb-backend/user-data-vault.sh b/examples/vault-dynamodb-backend/user-data-vault.sh index 6ff712c8..86b8047e 100644 --- a/examples/vault-dynamodb-backend/user-data-vault.sh +++ b/examples/vault-dynamodb-backend/user-data-vault.sh @@ -20,4 +20,7 @@ readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem" --dynamo-table "${dynamo_table_name}" \ --dynamo-region "${aws_region}" \ --tls-cert-file "$VAULT_TLS_CERT_FILE" \ - --tls-key-file "$VAULT_TLS_KEY_FILE" \ No newline at end of file + --tls-key-file "$VAULT_TLS_KEY_FILE" \ + --enable-s3-backend \ + --s3-bucket "${s3_bucket_name}" \ + --s3-bucket-region "${aws_region}" diff --git a/examples/vault-dynamodb-backend/variables.tf b/examples/vault-dynamodb-backend/variables.tf index 928f9b9e..b615bda3 100644 --- a/examples/vault-dynamodb-backend/variables.tf +++ b/examples/vault-dynamodb-backend/variables.tf @@ -65,3 +65,15 @@ variable "dynamo_write_capacity" { description = "Sets the DynamoDB write capacity for storage backend" default = 5 } + +variable "s3_bucket_name" { + description = "The name of an S3 bucket to create and use as a storage backend (if configured). Note: S3 bucket names must be *globally* unique." + type = string + default = "my-vault-bucket" +} + +variable "force_destroy_s3_bucket" { + description = "If you set this to true, when you run terraform destroy, this tells Terraform to delete all the objects in the S3 bucket used for backend storage (if configured). You should NOT set this to true in production or you risk losing all your data! This property is only here so automated tests of this module can clean up after themselves." + type = bool + default = false +} From bb94f43ae731871d75713de23fdbafaac8e041f7 Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Tue, 9 Feb 2021 23:05:59 +0100 Subject: [PATCH 11/14] Refactoring: get the initialized and unsealed vault cluster --- test/vault_helpers.go | 154 +++++++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 55 deletions(-) diff --git a/test/vault_helpers.go b/test/vault_helpers.go index d8d1aa37..0f5c1d02 100644 --- a/test/vault_helpers.go +++ b/test/vault_helpers.go @@ -61,10 +61,11 @@ func (cluster VaultCluster) Nodes() []ssh.Host { type VaultStatus int const ( - Leader VaultStatus = 200 - Standby = 429 - Uninitialized = 501 - Sealed = 503 + Leader VaultStatus = 200 + Standby = 429 + PerformanceStandby = 473 + Uninitialized = 501 + Sealed = 503 ) func teardownResources(t *testing.T, examplesDir string) { @@ -188,24 +189,100 @@ func initializeAndUnsealVaultCluster(t *testing.T, asgNameOutputVar string, sshU unsealVaultNode(t, cluster.Standby2, cluster.UnsealKeys) assertStatus(t, cluster.Standby2, Standby) + logger.Logf(t, "Successfully initialized and unsealed Vault Cluster: [Leader: %s, Standby1: %s , Standby2: %s]", cluster.Leader.Hostname, cluster.Standby1.Hostname, cluster.Standby2.Hostname) + return cluster } -// Find the initialized and unsealed Vault cluster, -// exit if cluster is not initialized and unsealed +// Find and return the initialized and unsealed Vault cluster, or exit if cluster is not initialized and unsealed: +// has not any of the expected staus codes: 200 for Leader node, 429 for Standby node or 473 for PerformanceStandby node (enterprise version) +// +// 1. Get the public IP addresses of the EC2 Instances in an Auto Scaling Group of the given name in the given region +// 2. SSH to each Vault nodes and get node status +// 3. Set Vault node status (Leader or Standby|PerformanceStandby) based on returned node status +// 4. Double check cluster node status func getInitializedAndUnsealedVaultCluster(t *testing.T, asgNameOutputVar string, sshUserName string, terraformOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) VaultCluster { - cluster := findVaultClusterNodes(t, asgNameOutputVar, sshUserName, terraformOptions, awsRegion, keyPair) + asgName := terraform.Output(t, terraformOptions, asgNameOutputVar) + nodeIpAddresses := getIpAddressesOfAsgInstances(t, asgName, awsRegion) + if len(nodeIpAddresses) != 3 { + t.Fatalf("Expected to get three IP addresses for Vault cluster, but got %d: %v", len(nodeIpAddresses), nodeIpAddresses) + } - establishConnectionToCluster(t, cluster) - isInitializedAndUnsealed, err := isVaultClusterInitializedAndUnsealed(t, cluster) - if err != nil { - require.NoError(t, err, "Failed to check is vault cluster is already initialized and unsealed") + // build vault cluster variable + // because findVaultClusterNodes method does not guarantee + // that node0 is Leader and node1 and node2 are Standby + cluster := VaultCluster{ + Leader: ssh.Host{ + Hostname: "", + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + }, + + Standby1: ssh.Host{ + Hostname: "", + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + }, + + Standby2: ssh.Host{ + Hostname: "", + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + }, } - // exit if cluster is not initialized and unsealed - if !isInitializedAndUnsealed { - t.Fatalf("Expected to find an initialized and unsealed cluster but it wasn't: [Leader: %s, Standby1:%s , Standby2: %s]", cluster.Leader.Hostname, cluster.Standby1.Hostname, cluster.Standby2.Hostname) + + // ssh each node and get its status + for _, node := range nodeIpAddresses { + if node != "" { + description := fmt.Sprintf("Trying to establish SSH connection to %s", node) + logger.Logf(t, description) + // connection to each of the Vault cluster nodes must be already working + // retrying only 3 times + maxRetries := 3 + sleepBetweenRetries := 10 * time.Second + host := ssh.Host{ + Hostname: node, + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + } + retry.DoWithRetry(t, description, maxRetries, sleepBetweenRetries, func() (string, error) { + return "", ssh.CheckSshConnectionE(t, host) + }) + // update leader and standby nodes in vault cluster variable + status, err := getNodeStatus(t, host) + if err != nil { + require.NoError(t, err, "Failed to check if vault cluster is already initialized and unsealed") + } + switch status { + case int(Leader): + cluster.Leader.Hostname = node + assertStatus(t, cluster.Leader, Leader) + case int(Standby): + if cluster.Standby1.Hostname == "" { + cluster.Standby1.Hostname = node + assertStatus(t, cluster.Standby1, Standby) + } else if cluster.Standby2.Hostname == "" { + cluster.Standby2.Hostname = node + } + // Managing Performance Standby Nodes status + // https://www.vaultproject.io/docs/enterprise/performance-standby#performance-standby-nodes + case int(PerformanceStandby): + if cluster.Standby1.Hostname == "" { + cluster.Standby1.Hostname = node + assertStatus(t, cluster.Standby1, PerformanceStandby) + } else if cluster.Standby2.Hostname == "" { + cluster.Standby2.Hostname = node + assertStatus(t, cluster.Standby2, PerformanceStandby) + } + default: + errMsg := fmt.Sprintf("error: Unexpected vault cluster node status %d", status) + require.NoError(t, errors.New(errMsg), "Failed to check if vault cluster is already initialized and unsealed") + } + } } + logger.Logf(t, "Retrieved Vault Cluster: [Leader: %s, Standby1: %s , Standby2: %s]", cluster.Leader.Hostname, cluster.Standby1.Hostname, cluster.Standby2.Hostname) + return cluster } @@ -441,17 +518,9 @@ func cleanupTlsCertFiles(tlsCert TlsCert) { os.Remove(tlsCert.PublicKeyPath) } -// Check the status of the given Vault node and ensure it matches the expected status. Note that we use curl to do the -// status check so we can ensure that TLS certificates work for curl (and not just the Vault client). +// Check the status of the given Vault node and ensure it matches the expected status. func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (string, error) { - curlCommand := "curl -s -o /dev/null -w '%{http_code}' https://127.0.0.1:8200/v1/sys/health" - logger.Logf(t, "Using curl to check status of Vault server %s: %s", host.Hostname, curlCommand) - - output, err := ssh.CheckSshCommandE(t, host, curlCommand) - if err != nil { - return "", err - } - status, err := strconv.Atoi(output) + status, err := getNodeStatus(t, host) if err != nil { return "", err } @@ -463,44 +532,19 @@ func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (strin } } -// Check if the given Vault cluster has been initialized and unsealed. -func isVaultClusterInitializedAndUnsealed(t *testing.T, cluster VaultCluster) (bool, error) { - leader, err := hasExpectedStatus(t, cluster.Leader, Leader) - if err != nil { - return false, err - } - standby1, err := hasExpectedStatus(t, cluster.Standby1, Standby) - if err != nil { - return false, err - } - standby2, err := hasExpectedStatus(t, cluster.Standby2, Standby) - if err != nil { - return false, err - } - if leader && standby1 && standby2 { - return true, nil - } else { - return false, nil - } -} - -// Check the status of the given Vault node and ensure it matches the expected status. -// Returns true if the status match, false otherwise. -func hasExpectedStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (bool, error) { +// Get the status of the given Vault node. Note that we use curl to do the status check so we can ensure that +// TLS certificates work for curl (and not just the Vault client). +func getNodeStatus(t *testing.T, host ssh.Host) (int, error) { curlCommand := "curl -s -o /dev/null -w '%{http_code}' https://127.0.0.1:8200/v1/sys/health" logger.Logf(t, "Using curl to check status of Vault server %s: %s", host.Hostname, curlCommand) output, err := ssh.CheckSshCommandE(t, host, curlCommand) if err != nil { - return false, err + return 0, err } status, err := strconv.Atoi(output) if err != nil { - return false, err - } - if status == int(expectedStatus) { - return true, nil - } else { - return false, nil + return 0, err } + return status, nil } From cfb7c453b6a8c24da0c7817407805c81df666506 Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Tue, 9 Feb 2021 23:06:12 +0100 Subject: [PATCH 12/14] Update tests --- test/go.sum | 2 ++ test/vault_cluster_dynamodb_backend_test.go | 8 +++++--- test/vault_cluster_enterprise_test.go | 9 ++++++++- test/vault_cluster_public_test.go | 1 + test/vault_cluster_s3_backend_test.go | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/test/go.sum b/test/go.sum index 7cabac8c..d89b9655 100644 --- a/test/go.sum +++ b/test/go.sum @@ -198,6 +198,7 @@ github.com/gruntwork-io/gruntwork-cli v0.5.1/go.mod h1:IBX21bESC1/LGoV7jhXKUnTQT github.com/gruntwork-io/terratest v0.28.15 h1:in1DRBq8/RjxMyb6Amr1SRrczOK/hGnPi+gQXOOtbZI= github.com/gruntwork-io/terratest v0.28.15/go.mod h1:PkVylPuUNmItkfOTwSiFreYA4FkanK8AluBuNeGxQOw= github.com/gruntwork-io/terratest v0.32.1 h1:Uho3H7VWD4tEulWov7pWW90V3XATLKxSh88AtrxTYvU= +github.com/gruntwork-io/terratest v0.32.3 h1:GSe/mkSQe0rD7Z92NKTUjDKg2FBuy0w82Ttd5gcK7kU= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -363,6 +364,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/test/vault_cluster_dynamodb_backend_test.go b/test/vault_cluster_dynamodb_backend_test.go index c2914c1b..a9c821c3 100644 --- a/test/vault_cluster_dynamodb_backend_test.go +++ b/test/vault_cluster_dynamodb_backend_test.go @@ -20,7 +20,6 @@ const VAR_DYNAMO_TABLE_NAME = "dynamo_table_name" // 3. Deploy that AMI using the example Terraform code // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it -// 6. Connect to the Vault cluster via the ELB func runVaultWithDynamoBackendClusterTest(t *testing.T, amiId string, awsRegion, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_DYNAMODB_BACKEND_PATH) @@ -36,10 +35,13 @@ func runVaultWithDynamoBackendClusterTest(t *testing.T, amiId string, awsRegion, }) test_structure.RunTestStage(t, "deploy", func() { + uniqueId := random.UniqueId() terraformVars := map[string]interface{}{ - VAR_DYNAMO_TABLE_NAME: fmt.Sprintf("vault-dynamo-test-%s", random.UniqueId()), + VAR_DYNAMO_TABLE_NAME: fmt.Sprintf("vault-dynamo-test-%s", uniqueId), + VAR_S3_BUCKET_NAME: s3BucketName(uniqueId), + VAR_FORCE_DESTROY_S3_BUCKET: true, } - deployCluster(t, amiId, awsRegion, examplesDir, random.UniqueId(), terraformVars) + deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) test_structure.RunTestStage(t, "validate", func() { diff --git a/test/vault_cluster_enterprise_test.go b/test/vault_cluster_enterprise_test.go index 4e4aad4f..ace11d85 100644 --- a/test/vault_cluster_enterprise_test.go +++ b/test/vault_cluster_enterprise_test.go @@ -59,11 +59,18 @@ func runVaultEnterpriseClusterTest(t *testing.T, amiId string, awsRegion string, deployCluster(t, amiId, awsRegion, examplesDir, uniqueId, terraformVars) }) + test_structure.RunTestStage(t, "initialize_unseal", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) + test_structure.RunTestStage(t, "validate", func() { terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) - cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + cluster := getInitializedAndUnsealedVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) testVaultUsesConsulForDns(t, cluster) checkEnterpriseInstall(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) }) diff --git a/test/vault_cluster_public_test.go b/test/vault_cluster_public_test.go index ec584021..6d3511e0 100644 --- a/test/vault_cluster_public_test.go +++ b/test/vault_cluster_public_test.go @@ -23,6 +23,7 @@ const VAULT_CLUSTER_PUBLIC_VAR_VAULT_DOMAIN_NAME = "vault_domain_name" // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it // 6. Connect to the Vault cluster via the ELB +// 7. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS func runVaultPublicClusterTest(t *testing.T, amiId string, awsRegion string, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, ".") diff --git a/test/vault_cluster_s3_backend_test.go b/test/vault_cluster_s3_backend_test.go index 473db2ac..f9887f7f 100644 --- a/test/vault_cluster_s3_backend_test.go +++ b/test/vault_cluster_s3_backend_test.go @@ -21,7 +21,7 @@ const VAR_FORCE_DESTROY_S3_BUCKET = "force_destroy_s3_bucket" // 3. Deploy that AMI using the example Terraform code // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it -// 6. Connect to the Vault cluster via the ELB +// 6. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS func runVaultWithS3BackendClusterTest(t *testing.T, amiId string, awsRegion, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_S3_BACKEND_PATH) From 96d70357c3d07f574e5ab6cc72a37f9c33f7a645 Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Wed, 10 Feb 2021 18:25:40 +0100 Subject: [PATCH 13/14] reviex: reverted variable format changes --- modules/run-vault/run-vault | 474 ++++++++++++++++++------------------ 1 file changed, 237 insertions(+), 237 deletions(-) diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index 28ab7c34..96955274 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -19,7 +19,7 @@ readonly DEFAULT_CONSUL_AGENT_SERVICE_REGISTRATION_ADDRESS="localhost:8500" readonly EC2_INSTANCE_METADATA_URL="http://169.254.169.254/latest/meta-data" readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -readonly SCRIPT_NAME="$(basename "${0}")" +readonly SCRIPT_NAME="$(basename "$0")" function print_usage { echo @@ -31,13 +31,13 @@ function print_usage { echo echo -e " --tls-cert-file\tSpecifies the path to the certificate for TLS. Required. To use a CA certificate, concatenate the primary certificate and the CA certificate together." echo -e " --tls-key-file\tSpecifies the path to the private key for the certificate. Required." - echo -e " --port\t\tThe port for Vault to listen on. Optional. Default is ${DEFAULT_PORT}." + echo -e " --port\t\tThe port for Vault to listen on. Optional. Default is $DEFAULT_PORT." echo -e " --cluster-port\tThe port for Vault to listen on for server-to-server requests. Optional. Default is --port + 1." - echo -e " --api-addr\t\tThe full address to use for Client Redirection when running Vault in HA mode. Defaults to \"https://[instance_ip]:${DEFAULT_PORT}\". Optional." + echo -e " --api-addr\t\tThe full address to use for Client Redirection when running Vault in HA mode. Defaults to \"https://[instance_ip]:$DEFAULT_PORT\". Optional." echo -e " --config-dir\t\tThe path to the Vault config folder. Optional. Default is the absolute path of '../config', relative to this script." echo -e " --bin-dir\t\tThe path to the folder with Vault binary. Optional. Default is the absolute path of the parent folder of this script." echo -e " --data-dir\t\tThe path to the Vault data folder. Optional. Default is the absolute path of '../data', relative to this script." - echo -e " --log-level\t\tThe log verbosity to use with Vault. Optional. Default is ${DEFAULT_LOG_LEVEL}." + echo -e " --log-level\t\tThe log verbosity to use with Vault. Optional. Default is $DEFAULT_LOG_LEVEL." echo -e " --systemd-stdout\t\tThe StandardOutput option of the systemd unit. Optional. If not configured, uses systemd's default (journal)." echo -e " --systemd-stderr\t\tThe StandardError option of the systemd unit. Optional. If not configured, uses systemd's default (inherit)." echo -e " --user\t\tThe user to run Vault as. Optional. Default is to use the owner of --config-dir." @@ -54,12 +54,12 @@ function print_usage { echo "Options for Vault Agent:" echo echo -e " --agent\t\t\tIf set, run in Vault Agent mode. If not set, run as a regular Vault server. Optional." - echo -e " --agent-vault-address\t\tThe hostname or IP address of the Vault server to connect to. Optional. Default is ${DEFAULT_AGENT_VAULT_ADDRESS}" - echo -e " --agent-vault-port\t\tThe port of the Vault server to connect to. Optional. Default is ${DEFAULT_PORT}" + echo -e " --agent-vault-address\t\tThe hostname or IP address of the Vault server to connect to. Optional. Default is $DEFAULT_AGENT_VAULT_ADDRESS" + echo -e " --agent-vault-port\t\tThe port of the Vault server to connect to. Optional. Default is $DEFAULT_PORT" echo -e " --agent-ca-cert-file\t\tSpecifies the path to a CA certificate to verify the Vault server's TLS certificate. Optional." echo -e " --agent-client-cert-file\tSpecifies the path to a certificate to use for TLS authentication to the Vault server. Optional." echo -e " --agent-client-key-file\tSpecifies the path to the private key for the client certificate used for TLS authentication to the Vault server. Optional." - echo -e " --agent-auth-mount-path\tThe Vault mount path to the auth method used for auto-auth. Optional. Defaults to ${DEFAULT_AGENT_AUTH_MOUNT_PATH}" + echo -e " --agent-auth-mount-path\tThe Vault mount path to the auth method used for auto-auth. Optional. Defaults to $DEFAULT_AGENT_AUTH_MOUNT_PATH" echo -e " --agent-auth-type\t\tThe Vault AWS auth type to use for auto-auth. Required. Must be either iam or ec2" echo -e " --agent-auth-role\t\tThe Vault role to authenticate against. Required." echo @@ -80,48 +80,48 @@ function print_usage { } function log { - local -r level="${1}" - local -r message="${2}" + local -r level="$1" + local -r message="$2" local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S") - >&2 echo -e "${timestamp} [${level}] [${SCRIPT_NAME}] ${message}" + >&2 echo -e "$timestamp [$level] [$SCRIPT_NAME] $message" } function log_info { - local -r message="${1}" - log "INFO" "${message}" + local -r message="$1" + log "INFO" "$message" } function log_warn { - local -r message="${1}" - log "WARN" "${message}" + local -r message="$1" + log "WARN" "$message" } function log_error { - local -r message="${1}" - log "ERROR" "${message}" + local -r message="$1" + log "ERROR" "$message" } # Based on code from: http://stackoverflow.com/a/16623897/483528 function strip_prefix { - local -r str="${1}" - local -r prefix="${2}" + local -r str="$1" + local -r prefix="$2" echo "${str#$prefix}" } function assert_not_empty { - local -r arg_name="${1}" - local -r arg_value="${2}" + local -r arg_name="$1" + local -r arg_value="$2" - if [[ -z "${arg_value}" ]]; then - log_error "The value for '${arg_name}' cannot be empty" + if [[ -z "$arg_value" ]]; then + log_error "The value for '$arg_name' cannot be empty" print_usage exit 1 fi } function lookup_path_in_instance_metadata { - local -r path="${1}" - curl --silent --location "${EC2_INSTANCE_METADATA_URL}/${path}/" + local -r path="$1" + curl --silent --location "$EC2_INSTANCE_METADATA_URL/$path/" } function get_instance_ip_address { @@ -129,10 +129,10 @@ function get_instance_ip_address { } function assert_is_installed { - local -r name="${1}" + local -r name="$1" - if [[ ! $(command -v ${name}) ]]; then - log_error "The binary '${name}' is required by this script but is not installed or in the system's PATH." + if [[ ! $(command -v $name) ]]; then + log_error "The binary '$name' is required by this script but is not installed or in the system's PATH." exit 1 fi } @@ -145,60 +145,60 @@ function get_vault_version { } function vault_version_at_least { - local -r config_path="${1}" - local -r ui_config="${2}" + local -r config_path="$1" + local -r ui_config="$2" VAULT_VERSION=$(get_vault_version) #This if statement will echo the current vault version and the minimum version required for ui support. #It then strips out the comments, sorts the two values, and chooses the top (least) one. - if [[ $(echo "${VAULT_VERSION} 0.10.0" | tr " " "\n" |sort --version-sort| head -n 1) = 0.10.0 ]] + if [[ $(echo "$VAULT_VERSION 0.10.0" | tr " " "\n" |sort --version-sort| head -n 1) = 0.10.0 ]] then - echo -e "${ui_config}" >> "${config_path}" + echo -e "$ui_config" >> "$config_path" else log_info "Vault 0.10.0 or greater is required for UI support." fi } function generate_vault_agent_config { - local -r config_dir="${1}" - local -r data_dir="${2}" - local -r vault_address="${3}" - local -r vault_port="${4}" - local -r ca_cert_file="${5}" - local -r client_cert_file="${6}" - local -r client_key_file="${7}" - local -r auth_mount_path="${8}" - local -r auth_type="${9}" + local -r config_dir="$1" + local -r data_dir="$2" + local -r vault_address="$3" + local -r vault_port="$4" + local -r ca_cert_file="$5" + local -r client_cert_file="$6" + local -r client_key_file="$7" + local -r auth_mount_path="$8" + local -r auth_type="$9" local -r auth_role="${10}" - local -r config_path="${config_dir}/${VAULT_CONFIG_FILE}" + local -r config_path="$config_dir/$VAULT_CONFIG_FILE" - log_info "Creating default Vault Agent config file in ${config_path}" + log_info "Creating default Vault Agent config file in $config_path" - local -r pid_config="pid_file = \"${data_dir}/$VAULT_PID_FILE\"" + local -r pid_config="pid_file = \"$data_dir/$VAULT_PID_FILE\"" local ca_cert_config="" - if [[ ! -z ${ca_cert_file} ]]; then + if [[ ! -z $ca_cert_file ]]; then ca_cert_config=$(cat < "${config_path}" - echo -e "${vault_config}" >> "${config_path}" - echo -e "${auto_auth_config}" >> "${config_path}" + echo -e "$pid_config" > "$config_path" + echo -e "$vault_config" >> "$config_path" + echo -e "$auto_auth_config" >> "$config_path" - chown "${user}:${user}" "${config_path}" + chown "$user:$user" "$config_path" } function generate_vault_config { - local -r tls_cert_file="${1}" - local -r tls_key_file="${2}" - local -r port="${3}" - local -r cluster_port="${4}" - local -r api_addr="${5}" - local -r config_dir="${6}" - local -r user="${7}" - local -r enable_s3_backend="${8}" - local -r s3_bucket="${9}" + local -r tls_cert_file="$1" + local -r tls_key_file="$2" + local -r port="$3" + local -r cluster_port="$4" + local -r api_addr="$5" + local -r config_dir="$6" + local -r user="$7" + local -r enable_s3_backend="$8" + local -r s3_bucket="$9" local -r s3_bucket_path="${10}" local -r s3_bucket_region="${11}" local -r consul_agent_service_registration_address="${12}" @@ -248,30 +248,30 @@ function generate_vault_config { local -r auto_unseal_kms_key_id="${17}" local -r auto_unseal_kms_key_region="${18}" local -r auto_unseal_endpoint="${19}" - local -r config_path="${config_dir}/${VAULT_CONFIG_FILE}" + local -r config_path="$config_dir/$VAULT_CONFIG_FILE" local instance_ip_address instance_ip_address=$(get_instance_ip_address) local auto_unseal_config="" - if [[ "${enable_auto_unseal}" == "true" ]]; then + if [[ "$enable_auto_unseal" == "true" ]]; then local endpoint="" - if [[ -n "${auto_unseal_endpoint}" ]]; then - endpoint="endpoint = \"${auto_unseal_endpoint}\"" + if [[ -n "$auto_unseal_endpoint" ]]; then + endpoint="endpoint = \"$auto_unseal_endpoint\"" fi auto_unseal_config=$(cat <> "${config_path}" - echo -e "${listener_config}" >> "${config_path}" - echo -e "${s3_config}" >> "${config_path}" - echo -e "${vault_storage_backend}" >> "${config_path}" - echo -e "${service_registration}" >> "${config_path}" + echo -e "$auto_unseal_config" >> "$config_path" + echo -e "$listener_config" >> "$config_path" + echo -e "$s3_config" >> "$config_path" + echo -e "$vault_storage_backend" >> "$config_path" + echo -e "$service_registration" >> "$config_path" - chown "${user}:${user}" "${config_path}" + chown "$user:$user" "$config_path" } function generate_systemd_config { - local -r systemd_config_path="${1}" - local -r vault_config_dir="${2}" - local -r vault_bin_dir="${3}" - local -r vault_log_level="${4}" - local -r vault_systemd_stdout="${5}" - local -r vault_systemd_stderr="${6}" - local -r vault_user="${7}" - local -r is_vault_agent="${8}" - local -r config_path="${config_dir}/${VAULT_CONFIG_FILE}" + local -r systemd_config_path="$1" + local -r vault_config_dir="$2" + local -r vault_bin_dir="$3" + local -r vault_log_level="$4" + local -r vault_systemd_stdout="$5" + local -r vault_systemd_stderr="$6" + local -r vault_user="$7" + local -r is_vault_agent="$8" + local -r config_path="$config_dir/$VAULT_CONFIG_FILE" local vault_description="HashiCorp Vault - A tool for managing secrets" local vault_command="server" @@ -367,18 +367,18 @@ function generate_systemd_config { if [[ "$is_vault_agent" == "true" ]]; then vault_command="agent" vault_description="HashiCorp Vault Agent" - vault_config_file_or_dir="${config_path}" + vault_config_file_or_dir="$config_path" fi - log_info "Creating systemd config file to run Vault in ${systemd_config_path}" + log_info "Creating systemd config file to run Vault in $systemd_config_path" local -r unit_config=$(cat < "${systemd_config_path}" - echo -e "${service_config}" >> "${systemd_config_path}" - echo -e "${log_config}" >> "${systemd_config_path}" - echo -e "${install_config}" >> "${systemd_config_path}" + echo -e "$unit_config" > "$systemd_config_path" + echo -e "$service_config" >> "$systemd_config_path" + echo -e "$log_config" >> "$systemd_config_path" + echo -e "$install_config" >> "$systemd_config_path" } function start_vault { @@ -443,20 +443,20 @@ function start_vault { # Based on: http://unix.stackexchange.com/a/7732/215969 function get_owner_of_path { - local -r path="${1}" - ls -ld "${path}" | awk '{print $3}' + local -r path="$1" + ls -ld "$path" | awk '{print $3}' } function run { local tls_cert_file="" local tls_key_file="" - local port="${DEFAULT_PORT}" + local port="$DEFAULT_PORT" local cluster_port="" local api_addr="" local config_dir="" local bin_dir="" local data_dir="" - local log_level="${DEFAULT_LOG_LEVEL}" + local log_level="$DEFAULT_LOG_LEVEL" local systemd_stdout="" local systemd_stderr="" local user="" @@ -470,12 +470,12 @@ function run { local dynamo_region="" local dynamo_table="" local agent="false" - local agent_vault_address="${DEFAULT_AGENT_VAULT_ADDRESS}" - local agent_vault_port="${DEFAULT_PORT}" + local agent_vault_address="$DEFAULT_AGENT_VAULT_ADDRESS" + local agent_vault_port="$DEFAULT_PORT" local agent_ca_cert_file="" local agent_client_cert_file="" local agent_client_key_file="" - local agent_auth_mount_path="${DEFAULT_AGENT_AUTH_MOUNT_PATH}" + local agent_auth_mount_path="$DEFAULT_AGENT_AUTH_MOUNT_PATH" local agent_auth_type="" local agent_auth_role="" local enable_auto_unseal="false" @@ -485,65 +485,65 @@ function run { local all_args=() while [[ $# > 0 ]]; do - local key="${1}" + local key="$1" - case "${key}" in + case "$key" in --tls-cert-file) - tls_cert_file="${2}" + tls_cert_file="$2" shift ;; --tls-key-file) - tls_key_file="${2}" + tls_key_file="$2" shift ;; --port) - assert_not_empty "${key}" "${2}" - port="${2}" + assert_not_empty "$key" "$2" + port="$2" shift ;; --cluster-port) - assert_not_empty "${key}" "${2}" - cluster_port="${2}" + assert_not_empty "$key" "$2" + cluster_port="$2" shift ;; --config-dir) - assert_not_empty "${key}" "${2}" - config_dir="${2}" + assert_not_empty "$key" "$2" + config_dir="$2" shift ;; --api-addr) - assert_not_empty "${key}" "${2}" - api_addr="${2}" + assert_not_empty "$key" "$2" + api_addr="$2" shift ;; --bin-dir) - assert_not_empty "${key}" "${2}" - bin_dir="${2}" + assert_not_empty "$key" "$2" + bin_dir="$2" shift ;; --data-dir) - assert_not_empty "${key}" "${2}" - data_dir="${2}" + assert_not_empty "$key" "$2" + data_dir="$2" shift ;; --log-level) - assert_not_empty "${key}" "${2}" - log_level="${2}" + assert_not_empty "$key" "$2" + log_level="$2" shift ;; --systemd-stdout) - assert_not_empty "${key}" "${2}" - systemd_stdout="${2}" + assert_not_empty "$key" "$2" + systemd_stdout="$2" shift ;; --systemd-stderr) - assert_not_empty "${key}" "${2}" - systemd_stderr="${2}" + assert_not_empty "$key" "$2" + systemd_stderr="$2" shift ;; --user) - assert_not_empty "${key}" "${2}" - user="${2}" + assert_not_empty "$key" "$2" + user="$2" shift ;; --skip-vault-config) @@ -553,87 +553,87 @@ function run { enable_s3_backend="true" ;; --s3-bucket) - s3_bucket="${2}" + s3_bucket="$2" shift ;; --s3-bucket-path) - s3_bucket_path="${2}" + s3_bucket_path="$2" shift ;; --s3-bucket-region) - s3_bucket_region="${2}" + s3_bucket_region="$2" shift ;; --consul-agent-service-registration-address) - assert_not_empty "${key}" "${2}" - consul_agent_service_registration_address="${2}" + assert_not_empty "$key" "$2" + consul_agent_service_registration_address="$2" shift ;; --enable-dynamo-backend) enable_dynamo_backend="true" ;; --dynamo-region) - dynamo_region="${2}" + dynamo_region="$2" shift ;; --dynamo-table) - dynamo_table="${2}" + dynamo_table="$2" shift ;; --agent) agent="true" ;; --agent-vault-address) - assert_not_empty "${key}" "${2}" - agent_vault_address="${2}" + assert_not_empty "$key" "$2" + agent_vault_address="$2" shift ;; --agent-vault-port) - assert_not_empty "${key}" "${2}" - agent_vault_port="${2}" + assert_not_empty "$key" "$2" + agent_vault_port="$2" shift ;; --agent-ca-cert-file) - assert_not_empty "${key}" "${2}" - agent_ca_cert_file="${2}" + assert_not_empty "$key" "$2" + agent_ca_cert_file="$2" shift ;; --agent-client-cert-file) - assert_not_empty "${key}" "${2}" - agent_client_cert_file="${2}" + assert_not_empty "$key" "$2" + agent_client_cert_file="$2" shift ;; --agent-client-key-file) - assert_not_empty "${key}" "${2}" - agent_client_key_file="${2}" + assert_not_empty "$key" "$2" + agent_client_key_file="$2" shift ;; --agent-auth-mount-path) - assert_not_empty "${key}" "${2}" - agent_auth_mount_path="${2}" + assert_not_empty "$key" "$2" + agent_auth_mount_path="$2" shift ;; --agent-auth-type) - agent_auth_type="${2}" + agent_auth_type="$2" shift ;; --agent-auth-role) - agent_auth_role="${2}" + agent_auth_role="$2" shift ;; --enable-auto-unseal) enable_auto_unseal="true" ;; --auto-unseal-kms-key-id) - auto_unseal_kms_key_id="${2}" + auto_unseal_kms_key_id="$2" shift ;; --auto-unseal-kms-key-region) - auto_unseal_kms_key_region="${2}" + auto_unseal_kms_key_region="$2" shift ;; --auto-unseal-endpoint) - auto_unseal_endpoint="${2}" + auto_unseal_endpoint="$2" shift ;; --help) @@ -641,7 +641,7 @@ function run { exit ;; *) - log_error "Unrecognized argument: ${key}" + log_error "Unrecognized argument: $key" print_usage exit 1 ;; @@ -651,23 +651,23 @@ function run { done # Required flags - if [[ "${agent}" == "true" ]]; then - assert_not_empty "--agent-auth-type" "${agent_auth_type}" - assert_not_empty "--agent-auth-role" "${agent_auth_role}" + if [[ "$agent" == "true" ]]; then + assert_not_empty "--agent-auth-type" "$agent_auth_type" + assert_not_empty "--agent-auth-role" "$agent_auth_role" else - assert_not_empty "--tls-cert-file" "${tls_cert_file}" - assert_not_empty "--tls-key-file" "${tls_key_file}" + assert_not_empty "--tls-cert-file" "$tls_cert_file" + assert_not_empty "--tls-key-file" "$tls_key_file" - if [[ "${enable_s3_backend}" == "true" ]]; then - assert_not_empty "--s3-bucket" "${s3_bucket}" - assert_not_empty "--s3-bucket-region" "${s3_bucket_region}" + if [[ "$enable_s3_backend" == "true" ]]; then + assert_not_empty "--s3-bucket" "$s3_bucket" + assert_not_empty "--s3-bucket-region" "$s3_bucket_region" assert_not_empty "--consul-agent-service-registration-address" "${consul_agent_service_registration_address}" fi fi - if [[ "${enable_dynamo_backend}" == "true" ]]; then - assert_not_empty "--dynamo-table" "${dynamo_table}" - assert_not_empty "--dynamo-region" "${dynamo_region}" + if [[ "$enable_dynamo_backend" == "true" ]]; then + assert_not_empty "--dynamo-table" "$dynamo_table" + assert_not_empty "--dynamo-region" "$dynamo_region" fi assert_is_installed "systemctl" @@ -675,80 +675,80 @@ function run { assert_is_installed "curl" assert_is_installed "jq" - if [[ -z "${config_dir}" ]]; then - config_dir=$(cd "${SCRIPT_DIR}/../config" && pwd) + if [[ -z "$config_dir" ]]; then + config_dir=$(cd "$SCRIPT_DIR/../config" && pwd) fi - # If ${systemd_stdout} and/or ${systemd_stderr} are empty, we leave them empty so that generate_systemd_config will use systemd's defaults (journal and inherit, respectively) + # If $systemd_stdout and/or $systemd_stderr are empty, we leave them empty so that generate_systemd_config will use systemd's defaults (journal and inherit, respectively) - if [[ -z "${bin_dir}" ]]; then - bin_dir=$(cd "${SCRIPT_DIR}/../bin" && pwd) + if [[ -z "$bin_dir" ]]; then + bin_dir=$(cd "$SCRIPT_DIR/../bin" && pwd) fi - if [[ -z "${data_dir}" ]]; then - data_dir=$(cd "${SCRIPT_DIR}/../data" && pwd) + if [[ -z "$data_dir" ]]; then + data_dir=$(cd "$SCRIPT_DIR/../data" && pwd) fi - if [[ -z "${user}" ]]; then - user=$(get_owner_of_path "${config_dir}") + if [[ -z "$user" ]]; then + user=$(get_owner_of_path "$config_dir") fi - if [[ -z "${cluster_port}" ]]; then - cluster_port=$(( ${port} + 1 )) + if [[ -z "$cluster_port" ]]; then + cluster_port=$(( $port + 1 )) fi - if [[ -z "${api_addr}" ]]; then - api_addr="https://$(get_instance_ip_address):${port}" + if [[ -z "$api_addr" ]]; then + api_addr="https://$(get_instance_ip_address):$port" fi - if [[ "${agent}" == "false" ]] && [[ "${enable_auto_unseal}" == "true" ]]; then + if [[ "$agent" == "false" ]] && [[ "$enable_auto_unseal" == "true" ]]; then log_info "The --enable-auto-unseal flag is set to true" - assert_not_empty "--auto-unseal-kms-key-id" "${auto_unseal_kms_key_id}" - assert_not_empty "--auto-unseal-kms-key-region" "${auto_unseal_kms_key_region}" + assert_not_empty "--auto-unseal-kms-key-id" "$auto_unseal_kms_key_id" + assert_not_empty "--auto-unseal-kms-key-region" "$auto_unseal_kms_key_region" fi - if [[ "${skip_vault_config}" == "true" ]]; then + if [[ "$skip_vault_config" == "true" ]]; then log_info "The --skip-vault-config flag is set, so will not generate a default Vault config file." else - if [[ "${agent}" == "true" ]]; then + if [[ "$agent" == "true" ]]; then log_info "Running as Vault agent (--agent flag is set)" generate_vault_agent_config \ - "${config_dir}" \ - "${data_dir}" \ - "${agent_vault_address}" \ - "${agent_vault_port}" \ - "${agent_ca_cert_file}" \ - "${agent_client_cert_file}" \ - "${agent_client_key_file}" \ - "${agent_auth_mount_path}" \ - "${agent_auth_type}" \ - "${agent_auth_role}" + "$config_dir" \ + "$data_dir" \ + "$agent_vault_address" \ + "$agent_vault_port" \ + "$agent_ca_cert_file" \ + "$agent_client_cert_file" \ + "$agent_client_key_file" \ + "$agent_auth_mount_path" \ + "$agent_auth_type" \ + "$agent_auth_role" else log_info "Running as Vault server" generate_vault_config \ - "${tls_cert_file}" \ - "${tls_key_file}" \ - "${port}" \ - "${cluster_port}" \ - "${api_addr}" \ - "${config_dir}" \ - "${user}" \ - "${enable_s3_backend}" \ - "${s3_bucket}" \ - "${s3_bucket_path}" \ - "${s3_bucket_region}" \ + "$tls_cert_file" \ + "$tls_key_file" \ + "$port" \ + "$cluster_port" \ + "$api_addr" \ + "$config_dir" \ + "$user" \ + "$enable_s3_backend" \ + "$s3_bucket" \ + "$s3_bucket_path" \ + "$s3_bucket_region" \ "${consul_agent_service_registration_address}" \ - "${enable_dynamo_backend}" \ - "${dynamo_region}" \ - "${dynamo_table}" \ - "${enable_auto_unseal}" \ - "${auto_unseal_kms_key_id}" \ - "${auto_unseal_kms_key_region}" \ - "${auto_unseal_endpoint}" + "$enable_dynamo_backend" \ + "$dynamo_region" \ + "$dynamo_table" \ + "$enable_auto_unseal" \ + "$auto_unseal_kms_key_id" \ + "$auto_unseal_kms_key_region" \ + "$auto_unseal_endpoint" fi fi - generate_systemd_config "${SYSTEMD_CONFIG_PATH}" "${config_dir}" "${bin_dir}" "${log_level}" "${systemd_stdout}" "${systemd_stderr}" "${user}" "${agent}" + generate_systemd_config "$SYSTEMD_CONFIG_PATH" "$config_dir" "$bin_dir" "$log_level" "$systemd_stdout" "$systemd_stderr" "$user" "$agent" start_vault } From c3bd168623c92fab62c36b49e8647e28859906aa Mon Sep 17 00:00:00 2001 From: Anouar Chattouna Date: Wed, 10 Feb 2021 18:51:13 +0100 Subject: [PATCH 14/14] reviex: reverted variable format changes --- modules/run-vault/run-vault | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/run-vault/run-vault b/modules/run-vault/run-vault index 96955274..edc880df 100755 --- a/modules/run-vault/run-vault +++ b/modules/run-vault/run-vault @@ -83,7 +83,7 @@ function log { local -r level="$1" local -r message="$2" local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S") - >&2 echo -e "$timestamp [$level] [$SCRIPT_NAME] $message" + >&2 echo -e "${timestamp} [${level}] [$SCRIPT_NAME] ${message}" } function log_info { @@ -131,7 +131,7 @@ function get_instance_ip_address { function assert_is_installed { local -r name="$1" - if [[ ! $(command -v $name) ]]; then + if [[ ! $(command -v ${name}) ]]; then log_error "The binary '$name' is required by this script but is not installed or in the system's PATH." exit 1 fi @@ -145,8 +145,8 @@ function get_vault_version { } function vault_version_at_least { - local -r config_path="$1" - local -r ui_config="$2" + local -r config_path=$1 + local -r ui_config=$2 VAULT_VERSION=$(get_vault_version) #This if statement will echo the current vault version and the minimum version required for ui support. @@ -698,7 +698,7 @@ function run { fi if [[ -z "$api_addr" ]]; then - api_addr="https://$(get_instance_ip_address):$port" + api_addr="https://$(get_instance_ip_address):${port}" fi if [[ "$agent" == "false" ]] && [[ "$enable_auto_unseal" == "true" ]]; then