Merge pull request #181 from stephencoe/dynamodb_backend

Adding support for DynamoDB backend

.circleci/config.yml

@@ -42,7 +42,7 @@ jobs:
           paths:
             - $HOME/.go_workspace/src/github.com/hashicorp/terraform-aws-vault/test/vendor
       - run: mkdir -p /tmp/logs
-      - run: run-go-tests --path test | tee /tmp/logs/all.log
+      - run: run-go-tests --path test --timeout 90m | tee /tmp/logs/all.log
       - run:
           command: terratest_log_parser --testlog /tmp/logs/all.log --outputdir /tmp/logs
           when: always

examples/vault-dynamodb-backend/README.md

@@ -0,0 +1,45 @@
+# Vault Cluster with DynamoDB backend example
+
+This folder shows an example of Terraform code to deploy a [Vault](https://www.vaultproject.io/) cluster in
+[AWS](https://aws.amazon.com/) using the [vault-cluster module](https://github.com/hashicorp/terraform-aws-vault/tree/master/modules/vault-cluster).
+The Vault cluster uses [DynamoDB](https://aws.amazon.com/dynamodb/) as a high-availability storage backend and [S3](https://aws.amazon.com/s3/)
+for durable storage, so this example also deploys a separate DynamoDB table
+
+This example creates a Vault cluster spread across the subnets in the default VPC of the AWS account. For an example of a Vault cluster
+that is publicly accessible, see [the root example](https://github.com/hashicorp/terraform-aws-vault/tree/master/examples/root-example).
+
+![Vault architecture](https://github.com/hashicorp/terraform-aws-vault/blob/master/_docs/architecture-with-dynamodb.png?raw=true)
+
+You will need to create an [Amazon Machine Image (AMI)](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html)
+that has Vault installed, which you can do using the [vault-consul-ami example](https://github.com/hashicorp/terraform-aws-vault/tree/master/examples/vault-consul-ami)).  
+
+For more info on how the Vault cluster works, check out the [vault-cluster](https://github.com/hashicorp/terraform-aws-vault/tree/master/modules/vault-cluster) documentation.
+
+**Note**: To keep this example as simple to deploy and test as possible, it deploys the Vault cluster into your default
+VPC and default subnets, some of which might be publicly accessible. This is OK for learning and experimenting, but for
+production usage, we strongly recommend deploying the Vault cluster into the private subnets of a custom VPC.
+
+
+
+
+## Quick start
+
+To deploy a Vault Cluster:
+
+1. `git clone` this repo to your computer.
+1. Optional: build a Vault and Consul AMI. See the [vault-consul-ami
+   example](https://github.com/hashicorp/terraform-aws-vault/tree/master/examples/vault-consul-ami) documentation for
+   instructions. Make sure to note down the ID of the AMI.
+1. Install [Terraform](https://www.terraform.io/).
+1. Open `variables.tf`, set the environment variables specified at the top of the file, and fill in any other variables that
+   don't have a default. If you built a custom AMI, put the AMI ID into the `ami_id` variable. Otherwise, one of our
+   public example AMIs will be used by default. These AMIs are great for learning/experimenting, but are NOT
+   recommended for production use.
+1. Run `terraform init`.
+1. Run `terraform apply`.
+1. Run the [vault-examples-helper.sh script](https://github.com/hashicorp/terraform-aws-vault/tree/master/examples/vault-examples-helper/vault-examples-helper.sh) to
+   print out the IP addresses of the Vault servers and some example commands you can run to interact with the cluster:
+   `../vault-examples-helper/vault-examples-helper.sh`.
+
+To see how to connect to the Vault cluster, initialize it, and start reading and writing secrets, head over to the
+[How do you use the Vault cluster?](https://github.com/hashicorp/terraform-aws-vault/tree/master/modules/vault-cluster#how-do-you-use-the-vault-cluster) docs.

examples/vault-dynamodb-backend/dynamodb/main.tf

@@ -0,0 +1,17 @@
+resource "aws_dynamodb_table" "vault_dynamo" {
+  name           = var.table_name
+  hash_key       = "Path"
+  range_key      = "Key"
+  read_capacity  = var.read_capacity
+  write_capacity = var.write_capacity
+
+  attribute {
+    name = "Path"
+    type = "S"
+  }
+
+  attribute {
+    name = "Key"
+    type = "S"
+  }
+}

examples/vault-dynamodb-backend/dynamodb/variables.tf

@@ -0,0 +1,18 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# You must provide a value for each of these parameters.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "table_name" {
+  description = "The name of the Dynamo Table to create and use as a storage backend."
+}
+
+variable "read_capacity" {
+  description = "Sets the DynamoDB read capacity for storage backend"
+  default     = 5
+}
+
+variable "write_capacity" {
+  description = "Sets the DynamoDB write capacity for storage backend"
+  default     = 5
+}

examples/vault-dynamodb-backend/main.tf

@@ -0,0 +1,76 @@
+# ----------------------------------------------------------------------------------------------------------------------
+# REQUIRE A SPECIFIC TERRAFORM VERSION OR HIGHER
+# This module has been updated with 0.12 syntax, which means it is no longer compatible with any versions below 0.12.
+# ----------------------------------------------------------------------------------------------------------------------
+terraform {
+  required_version = ">= 0.12"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# DEPLOY THE VAULT SERVER CLUSTER
+# ---------------------------------------------------------------------------------------------------------------------
+
+module "backend" {
+  source         = "./dynamodb"
+  table_name     = var.dynamo_table_name
+  read_capacity  = var.dynamo_read_capacity
+  write_capacity = var.dynamo_write_capacity
+}
+
+module "vault_cluster" {
+  # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you
+  # to a specific version of the modules, such as the following example:
+  # source = "github.com/hashicorp/terraform-aws-vault.git//modules/vault-cluster?ref=v0.0.1"
+  source = "../../modules/vault-cluster"
+
+  cluster_name  = var.vault_cluster_name
+  cluster_size  = var.vault_cluster_size
+  instance_type = var.vault_instance_type
+
+  ami_id    = var.ami_id
+  user_data = data.template_file.user_data_vault_cluster.rendered
+
+  vpc_id     = data.aws_vpc.default.id
+  subnet_ids = data.aws_subnet_ids.default.ids
+
+  # To make testing easier, we allow requests from any IP address here but in a production deployment, we *strongly*
+  # recommend you limit this to the IP address ranges of known, trusted servers inside your VPC.
+
+  allowed_ssh_cidr_blocks              = ["0.0.0.0/0"]
+  allowed_inbound_cidr_blocks          = ["0.0.0.0/0"]
+  allowed_inbound_security_group_ids   = []
+  allowed_inbound_security_group_count = 0
+  ssh_key_name                         = var.ssh_key_name
+
+  enable_dynamo_backend = true
+  dynamo_table_name = var.dynamo_table_name
+}
+
+data "template_file" "user_data_vault_cluster" {
+  template = file("${path.module}/user-data-vault.sh")
+
+  vars = {
+    aws_region        = data.aws_region.current.name
+    dynamo_table_name = var.dynamo_table_name
+  }
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# DEPLOY THE CLUSTERS IN THE DEFAULT VPC AND AVAILABILITY ZONES
+# Using the default VPC and subnets makes this example easy to run and test, but it means Vault is
+# accessible from the public Internet. In a production deployment, we strongly recommend deploying into a custom VPC
+# and private subnets.
+# ---------------------------------------------------------------------------------------------------------------------
+
+data "aws_vpc" "default" {
+  default = var.vpc_id == null ? true : false
+  id      = var.vpc_id
+}
+
+data "aws_subnet_ids" "default" {
+  vpc_id = data.aws_vpc.default.id
+}
+
+data "aws_region" "current" {
+}
+

examples/vault-dynamodb-backend/outputs.tf

@@ -0,0 +1,39 @@
+output "asg_name_vault_cluster" {
+  value = module.vault_cluster.asg_name
+}
+
+output "launch_config_name_vault_cluster" {
+  value = module.vault_cluster.launch_config_name
+}
+
+output "iam_role_arn_vault_cluster" {
+  value = module.vault_cluster.iam_role_arn
+}
+
+output "iam_role_id_vault_cluster" {
+  value = module.vault_cluster.iam_role_id
+}
+
+output "security_group_id_vault_cluster" {
+  value = module.vault_cluster.security_group_id
+}
+
+output "aws_region" {
+  value = data.aws_region.current.name
+}
+
+output "vault_servers_cluster_tag_key" {
+  value = module.vault_cluster.cluster_tag_key
+}
+
+output "vault_servers_cluster_tag_value" {
+  value = module.vault_cluster.cluster_tag_value
+}
+
+output "ssh_key_name" {
+  value = var.ssh_key_name
+}
+
+output "vault_cluster_size" {
+  value = var.vault_cluster_size
+}

examples/vault-dynamodb-backend/user-data-vault.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# This script is meant to be run in the User Data of each EC2 Instance while it's booting. The script uses the
+# run-vault script to configure and start Vault in server mode.  
+# Note that this script assumes it's running in an AMI built from the Packer template in 
+# examples/vault-consul-ami/vault-consul.json.
+
+set -e
+
+# Send the log output from this script to user-data.log, syslog, and the console
+# From: https://alestic.com/2010/12/ec2-user-data-output/
+exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
+
+# The Packer template puts the TLS certs in these file paths
+readonly VAULT_TLS_CERT_FILE="/opt/vault/tls/vault.crt.pem"
+readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem"
+
+# The variables below are filled in via Terraform interpolation
+/opt/vault/bin/run-vault \
+  --enable-dynamo-backend \
+  --dynamo-table "${dynamo_table_name}" \
+  --dynamo-region "${aws_region}" \
+  --tls-cert-file "$VAULT_TLS_CERT_FILE" \
+  --tls-key-file "$VAULT_TLS_KEY_FILE"

examples/vault-dynamodb-backend/variables.tf

@@ -0,0 +1,67 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# ENVIRONMENT VARIABLES
+# Define these secrets as environment variables
+# ---------------------------------------------------------------------------------------------------------------------
+
+# AWS_ACCESS_KEY_ID
+# AWS_SECRET_ACCESS_KEY
+# AWS_DEFAULT_REGION
+
+# ---------------------------------------------------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+# You must provide a value for each of these parameters.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "ami_id" {
+  description = "The ID of the AMI to run in the cluster. This should be an AMI built from the Packer template under examples/vault-consul-ami/vault-consul.json."
+  type        = string
+}
+
+variable "ssh_key_name" {
+  description = "The name of an EC2 Key Pair that can be used to SSH to the EC2 Instances in this cluster. Set to an empty string to not associate a Key Pair."
+  type        = string
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+# These parameters have reasonable defaults.
+# ---------------------------------------------------------------------------------------------------------------------
+
+variable "vault_cluster_name" {
+  description = "What to name the Vault server cluster and all of its associated resources"
+  type        = string
+  default     = "vault-dynamo-example"
+}
+
+variable "vault_cluster_size" {
+  description = "The number of Vault server nodes to deploy. We strongly recommend using 3 or 5."
+  type        = number
+  default     = 3
+}
+
+variable "vault_instance_type" {
+  description = "The type of EC2 Instance to run in the Vault ASG"
+  type        = string
+  default     = "t2.micro"
+}
+
+variable "vpc_id" {
+  description = "The ID of the VPC to deploy into. Leave an empty string to use the Default VPC in this region."
+  type        = string
+  default     = null
+}
+
+variable "dynamo_table_name" {
+  description = "The name of the Dynamo Table to create and use as a storage backend. Only used if 'enable_dynamo_backend' is set to true."
+  default     = "my-vault-table"
+}
+
+variable "dynamo_read_capacity" {
+  description = "Sets the DynamoDB read capacity for storage backend"
+  default     = 5
+}
+
+variable "dynamo_write_capacity" {
+  description = "Sets the DynamoDB write capacity for storage backend"
+  default     = 5
+}