From 4966729c2519b99abceb9a15bc0d11baae7f6e6c Mon Sep 17 00:00:00 2001 From: Etiene Dalcol Date: Wed, 29 Aug 2018 17:39:55 +0100 Subject: [PATCH 1/2] Add test for vault enterprise install --- examples/vault-consul-ami/vault-consul.json | 8 +- modules/install-vault/install-vault | 10 +- test/terratest_helpers.go | 29 +++++- test/vault_cluster_enterprise_test.go | 16 +++ test/vault_helpers.go | 103 ++++++++++++++++++++ 5 files changed, 156 insertions(+), 10 deletions(-) create mode 100644 test/vault_cluster_enterprise_test.go diff --git a/examples/vault-consul-ami/vault-consul.json b/examples/vault-consul-ami/vault-consul.json index c082f3b8..0eb6c2a3 100644 --- a/examples/vault-consul-ami/vault-consul.json +++ b/examples/vault-consul-ami/vault-consul.json @@ -5,8 +5,8 @@ "vault_version": "0.10.4", "consul_module_version": "v0.3.7", "consul_version": "1.2.2", - "consul_download_url": "", - "vault_download_url": "", + "consul_download_url": "{{env `CONSUL_DOWNLOAD_URL`}}", + "vault_download_url": "{{env `VAULT_DOWNLOAD_URL`}}", "github_oauth_token": "{{env `GITHUB_OAUTH_TOKEN`}}", "ca_public_key_path": null, "tls_public_key_path": null, @@ -59,7 +59,7 @@ },{ "type": "shell", "inline": [ - "if [[ -n '{{user `vault_download_url`}}' ]]; then", + "if test -n \"{{user `vault_download_url`}}\"; then", " /tmp/terraform-aws-vault/modules/install-vault/install-vault --download-url {{user `vault_download_url`}};", "else", " /tmp/terraform-aws-vault/modules/install-vault/install-vault --version {{user `vault_version`}};", @@ -104,7 +104,7 @@ "type": "shell", "inline": [ "git clone --branch {{user `consul_module_version`}} https://{{user `github_oauth_token`}}@github.com/hashicorp/terraform-aws-consul.git /tmp/terraform-aws-consul", - "if [[ -n '{{user `consul_download_url`}}' ]]; then", + "if test -n \"{{user `consul_download_url`}}\"; then", " /tmp/terraform-aws-consul/modules/install-consul/install-consul --download-url {{user `consul_download_url`}};", "else", " /tmp/terraform-aws-consul/modules/install-consul/install-consul --version {{user `consul_version`}};", diff --git a/modules/install-vault/install-vault b/modules/install-vault/install-vault index b468f31e..f68b47a6 100755 --- a/modules/install-vault/install-vault +++ b/modules/install-vault/install-vault @@ -204,7 +204,7 @@ function fetch_binary { download_url="https://releases.hashicorp.com/vault/${version}/vault_${version}_linux_amd64.zip" fi - log_info "Downloading Vault from $download_url to $DOWNLOAD_PACKAGE_PATH" + log_info "Downloading Vault to $DOWNLOAD_PACKAGE_PATH" curl -o "$DOWNLOAD_PACKAGE_PATH" "$download_url" --location --silent --fail --show-error } @@ -296,7 +296,13 @@ function install { install_binary "$path" "$user" configure_mlock - log_info "Vault install complete!" + + if command -v vault; then + log_info "Vault install complete!" + else + log_info "Could not find vault command. Aborting."; + exit 1; + fi } install "$@" diff --git a/test/terratest_helpers.go b/test/terratest_helpers.go index 0a169092..352b8555 100644 --- a/test/terratest_helpers.go +++ b/test/terratest_helpers.go @@ -2,6 +2,7 @@ package test import ( "testing" + "github.com/gruntwork-io/terratest/modules/packer" "github.com/gruntwork-io/terratest/modules/test-structure" ) @@ -10,6 +11,7 @@ const AMI_VAR_AWS_REGION = "aws_region" const AMI_VAR_CA_PUBLIC_KEY = "ca_public_key_path" const AMI_VAR_TLS_PUBLIC_KEY = "tls_public_key_path" const AMI_VAR_TLS_PRIVATE_KEY = "tls_private_key_path" +const AMI_VAR_VAULT_DOWNLOAD_URL = "VAULT_DOWNLOAD_URL" const SAVED_TLS_CERT = "TlsCert" @@ -17,11 +19,11 @@ const SAVED_TLS_CERT = "TlsCert" func buildAmi(t *testing.T, packerTemplatePath string, packerBuildName string, tlsCert TlsCert, awsRegion string) string { options := &packer.Options{ Template: packerTemplatePath, - Only: packerBuildName, + Only: packerBuildName, Vars: map[string]string{ - AMI_VAR_AWS_REGION: awsRegion, - AMI_VAR_CA_PUBLIC_KEY: tlsCert.CAPublicKeyPath, - AMI_VAR_TLS_PUBLIC_KEY: tlsCert.PublicKeyPath, + AMI_VAR_AWS_REGION: awsRegion, + AMI_VAR_CA_PUBLIC_KEY: tlsCert.CAPublicKeyPath, + AMI_VAR_TLS_PUBLIC_KEY: tlsCert.PublicKeyPath, AMI_VAR_TLS_PRIVATE_KEY: tlsCert.PrivateKeyPath, }, } @@ -29,6 +31,25 @@ func buildAmi(t *testing.T, packerTemplatePath string, packerBuildName string, t return packer.BuildAmi(t, options) } +// Use Packer to build the AMI in the given packer template, with the given build name, and return the AMI's ID +func buildAmiWithDownloadEnv(t *testing.T, packerTemplatePath string, packerBuildName string, tlsCert TlsCert, awsRegion string, vaultDownloadUrl string) string { + options := &packer.Options{ + Template: packerTemplatePath, + Only: packerBuildName, + Vars: map[string]string{ + AMI_VAR_AWS_REGION: awsRegion, + AMI_VAR_CA_PUBLIC_KEY: tlsCert.CAPublicKeyPath, + AMI_VAR_TLS_PUBLIC_KEY: tlsCert.PublicKeyPath, + AMI_VAR_TLS_PRIVATE_KEY: tlsCert.PrivateKeyPath, + }, + Env: map[string]string{ + AMI_VAR_VAULT_DOWNLOAD_URL: vaultDownloadUrl, + }, + } + + return packer.BuildAmi(t, options) +} + func saveTlsCert(t *testing.T, testFolder string, tlsCert TlsCert) { test_structure.SaveTestData(t, test_structure.FormatTestDataPath(testFolder, SAVED_TLS_CERT), tlsCert) } diff --git a/test/vault_cluster_enterprise_test.go b/test/vault_cluster_enterprise_test.go new file mode 100644 index 00000000..7c6fc676 --- /dev/null +++ b/test/vault_cluster_enterprise_test.go @@ -0,0 +1,16 @@ +package test + +import ( + "os" + "testing" +) + +func TestVaultClusterEnterpriseWithUbuntuAmi(t *testing.T) { + t.Parallel() + runVaultEnterpriseClusterTest(t, "ubuntu16-ami", "ubuntu", os.Getenv("VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL")) +} + +func TestVaultClusterEnterpriseWithAmazonLinuxAmi(t *testing.T) { + t.Parallel() + runVaultEnterpriseClusterTest(t, "amazon-linux-ami", "ec2-user", os.Getenv("VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL")) +} diff --git a/test/vault_helpers.go b/test/vault_helpers.go index a205f6d0..95342ec8 100644 --- a/test/vault_helpers.go +++ b/test/vault_helpers.go @@ -333,6 +333,81 @@ func runVaultWithS3BackendClusterTest(t *testing.T, packerBuildName string, sshU }) } +// Test the Vault enterprise cluster example by: +// +// 1. Copy the code in this repo to a temp folder so tests on the Terraform code can run in parallel without the +// state files overwriting each other. +// 2. Build the AMI in the vault-consul-ami example with the given build name and the enterprise package +// 3. Deploy that AMI using the example Terraform code +// 4. SSH to a Vault node and initialize the Vault cluster +// 5. SSH to each Vault node and unseal it +// 5. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS +func runVaultEnterpriseClusterTest(t *testing.T, packerBuildName string, sshUserName string, vaultDownloadUrl string) { + examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_PRIVATE_PATH) + + defer test_structure.RunTestStage(t, "teardown", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + terraform.Destroy(t, terraformOptions) + + amiId := test_structure.LoadAmiId(t, examplesDir) + awsRegion := test_structure.LoadString(t, examplesDir, SAVED_AWS_REGION) + aws.DeleteAmi(t, awsRegion, amiId) + + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + aws.DeleteEC2KeyPair(t, keyPair) + + tlsCert := loadTlsCert(t, examplesDir) + cleanupTlsCertFiles(tlsCert) + }) + + test_structure.RunTestStage(t, "setup_ami", func() { + awsRegion := aws.GetRandomRegion(t, nil, nil) + test_structure.SaveString(t, examplesDir, SAVED_AWS_REGION, awsRegion) + + tlsCert := generateSelfSignedTlsCert(t) + saveTlsCert(t, examplesDir, tlsCert) + + amiId := buildAmiWithDownloadEnv(t, AMI_EXAMPLE_PATH, packerBuildName, tlsCert, awsRegion, vaultDownloadUrl) + test_structure.SaveAmiId(t, examplesDir, amiId) + }) + + test_structure.RunTestStage(t, "deploy", func() { + uniqueId := random.UniqueId() + amiId := test_structure.LoadAmiId(t, examplesDir) + awsRegion := test_structure.LoadString(t, examplesDir, SAVED_AWS_REGION) + + keyPair := aws.CreateAndImportEC2KeyPair(t, awsRegion, uniqueId) + test_structure.SaveEc2KeyPair(t, examplesDir, keyPair) + + terraformOptions := &terraform.Options{ + TerraformDir: examplesDir, + Vars: map[string]interface{}{ + VAR_AMI_ID: amiId, + VAR_VAULT_CLUSTER_NAME: fmt.Sprintf("vault-test-%s", uniqueId), + VAR_CONSUL_CLUSTER_NAME: fmt.Sprintf("consul-test-%s", uniqueId), + VAR_CONSUL_CLUSTER_TAG_KEY: fmt.Sprintf("consul-test-%s", uniqueId), + VAR_SSH_KEY_NAME: keyPair.Name, + }, + EnvVars: map[string]string{ + ENV_VAR_AWS_REGION: awsRegion, + }, + } + test_structure.SaveTerraformOptions(t, examplesDir, terraformOptions) + + terraform.InitAndApply(t, terraformOptions) + }) + + test_structure.RunTestStage(t, "validate", func() { + terraformOptions := test_structure.LoadTerraformOptions(t, examplesDir) + awsRegion := test_structure.LoadString(t, examplesDir, SAVED_AWS_REGION) + keyPair := test_structure.LoadEc2KeyPair(t, examplesDir) + + cluster := initializeAndUnsealVaultCluster(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + testVaultUsesConsulForDns(t, cluster) + checkEnterpriseInstall(t, OUTPUT_VAULT_CLUSTER_ASG_NAME, sshUserName, terraformOptions, awsRegion, keyPair) + }) +} + // Initialize the Vault cluster and unseal each of the nodes by connecting to them over SSH and executing Vault // commands. The reason we use SSH rather than using the Vault client remotely is we want to verify that the // self-signed TLS certificate is properly configured on each server so when you're on that server, you don't @@ -598,3 +673,31 @@ func checkStatus(t *testing.T, host ssh.Host, expectedStatus VaultStatus) (strin return "", fmt.Errorf("Expected status code %d for host %s, but got %d", int(expectedStatus), host.Hostname, status) } } + +// Check if the enterprise version of consul and vault is installed +func checkEnterpriseInstall(t *testing.T, asgNameOutputVar string, sshUserName string, terratestOptions *terraform.Options, awsRegion string, keyPair *aws.Ec2Keypair) { + asgName := terraform.OutputRequired(t, terratestOptions, asgNameOutputVar) + nodeIpAddresses := getIpAddressesOfAsgInstances(t, asgName, awsRegion) + + host := ssh.Host{ + Hostname: nodeIpAddresses[0], + SshUserName: sshUserName, + SshKeyPair: keyPair.KeyPair, + } + + maxRetries := 10 + sleepBetweenRetries := 10 * time.Second + + output := retry.DoWithRetry(t, "Check Enterprise Install", maxRetries, sleepBetweenRetries, func() (string, error) { + out, err := ssh.CheckSshCommandE(t, host, "vault --version") + if err != nil { + return "", fmt.Errorf("Error running vault command: %s\n", err) + } + + return out, nil + }) + + if !strings.Contains(output, "+ent") { + t.Fatalf("This vault package is not the enterprise version.\n") + } +} From 28212b8b826c22ede48bf5893ebef1f43764814a Mon Sep 17 00:00:00 2001 From: Etiene Dalcol Date: Thu, 30 Aug 2018 13:29:22 +0100 Subject: [PATCH 2/2] Bumps consul module version and checks env var for enterprise test --- examples/vault-consul-ami/vault-consul.json | 2 +- test/vault_cluster_enterprise_test.go | 15 +++++++++++++-- test/vault_helpers.go | 5 +++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/examples/vault-consul-ami/vault-consul.json b/examples/vault-consul-ami/vault-consul.json index 0eb6c2a3..4e845bb0 100644 --- a/examples/vault-consul-ami/vault-consul.json +++ b/examples/vault-consul-ami/vault-consul.json @@ -3,7 +3,7 @@ "variables": { "aws_region": "us-east-1", "vault_version": "0.10.4", - "consul_module_version": "v0.3.7", + "consul_module_version": "v0.3.10", "consul_version": "1.2.2", "consul_download_url": "{{env `CONSUL_DOWNLOAD_URL`}}", "vault_download_url": "{{env `VAULT_DOWNLOAD_URL`}}", diff --git a/test/vault_cluster_enterprise_test.go b/test/vault_cluster_enterprise_test.go index 7c6fc676..553c7321 100644 --- a/test/vault_cluster_enterprise_test.go +++ b/test/vault_cluster_enterprise_test.go @@ -7,10 +7,21 @@ import ( func TestVaultClusterEnterpriseWithUbuntuAmi(t *testing.T) { t.Parallel() - runVaultEnterpriseClusterTest(t, "ubuntu16-ami", "ubuntu", os.Getenv("VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL")) + runVaultEnterpriseClusterTest(t, "ubuntu16-ami", "ubuntu", getUrlFromEnv(t)) } func TestVaultClusterEnterpriseWithAmazonLinuxAmi(t *testing.T) { t.Parallel() - runVaultEnterpriseClusterTest(t, "amazon-linux-ami", "ec2-user", os.Getenv("VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL")) + runVaultEnterpriseClusterTest(t, "amazon-linux-ami", "ec2-user", getUrlFromEnv(t)) +} + +// To test this on circle ci you need a url set as an environment variable, VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL +// which you would also have to set locally if you want to run this test locally. +// The reason is to prevent the actual url from being visible on code and logs +func getUrlFromEnv(t *testing.T) string { + url := os.Getenv("VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL") + if url == "" { + t.Fatalf("Please set the environment variable VAULT_AMI_TEMPLATE_VAR_DOWNLOAD_URL.\n") + } + return url } diff --git a/test/vault_helpers.go b/test/vault_helpers.go index 95342ec8..c5162964 100644 --- a/test/vault_helpers.go +++ b/test/vault_helpers.go @@ -87,7 +87,7 @@ const ( // 3. Deploy that AMI using the example Terraform code // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it -// 5. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS +// 6. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS func runVaultPrivateClusterTest(t *testing.T, packerBuildName string, sshUserName string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_PRIVATE_PATH) @@ -341,7 +341,8 @@ func runVaultWithS3BackendClusterTest(t *testing.T, packerBuildName string, sshU // 3. Deploy that AMI using the example Terraform code // 4. SSH to a Vault node and initialize the Vault cluster // 5. SSH to each Vault node and unseal it -// 5. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS +// 6. SSH to a Vault node and make sure you can communicate with the nodes via Consul-managed DNS +// 7. SSH to a Vault node and check if Vault enterprise is installed properly func runVaultEnterpriseClusterTest(t *testing.T, packerBuildName string, sshUserName string, vaultDownloadUrl string) { examplesDir := test_structure.CopyTerraformFolderToTemp(t, REPO_ROOT, VAULT_CLUSTER_PRIVATE_PATH)