Merge pull request #111 from hashicorp/test/intermittent-failures

Test/intermittent failures

examples/vault-consul-ami/vault-consul.json

@@ -2,9 +2,9 @@
   "min_packer_version": "0.12.0",
   "variables": {
     "aws_region": "us-east-1",
-    "vault_version": "0.10.4",
-    "consul_module_version": "v0.3.10",
-    "consul_version": "1.2.2",
+    "vault_version": "0.11.5",
+    "consul_module_version": "v0.4.2",
+    "consul_version": "1.3.1",
     "consul_download_url": "{{env `CONSUL_DOWNLOAD_URL`}}",
     "vault_download_url": "{{env `VAULT_DOWNLOAD_URL`}}",
     "ca_public_key_path": null,

examples/vault-ec2-auth/outputs.tf

@@ -2,6 +2,10 @@ output "auth_client_public_ip" {
   value = "${aws_instance.example_auth_to_vault.public_ip}"
 }
 
+output "auth_client_instance_id" {
+  value = "${aws_instance.example_auth_to_vault.id}"
+}
+
 output "asg_name_vault_cluster" {
   value = "${module.vault_cluster.asg_name}"
 }

examples/vault-ec2-auth/user-data-auth-client.sh

@@ -12,27 +12,37 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 # These variables are passed in via Terraform template interpolation
 /opt/consul/bin/run-consul --client --cluster-tag-key "${consul_cluster_tag_key}" --cluster-tag-value "${consul_cluster_tag_value}"
 
-# run-consul is running on the background, so we have to wait for it and
-# we also have to for wait for vault server to be booted and unsealed before it can accept this request
-# so in case this fails we retry.
-function retry_login {
-  local readonly data=$1
-  local readonly url=$2
+# Log the given message. All logs are written to stderr with a timestamp.
+function log {
+ local -r message="$1"
+ local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+ >&2 echo -e "$timestamp $message"
+}
+
+# A retry function that attempts to run a command a number of times and returns the output
+function retry {
+  local -r cmd="$1"
+  local -r description="$2"
 
   for i in $(seq 1 30); do
-    echo "Attempting to authenticate to Vault..."
+    log "$description"
+
     # The boolean operations with the exit status are there to temporarily circumvent the "set -e" at the
     # beginning of this script which exits the script immediatelly for error status while not losing the exit status code
-    # The important bit is the curl request
-    login_output=$(curl --request POST --data "$data" "$url") && exit_status=0 || exit_status=$?
-    if [[ $exit_status -eq 0 ]]; then
+    output=$(eval "$cmd") && exit_status=0 || exit_status=$?
+    errors=$(echo "$output") | grep '^{' | jq -r .errors
+
+    log "$output"
+
+    if [[ $exit_status -eq 0 && -n "$output" && -z "$errors" ]]; then
+      echo "$output"
       return
     fi
-    echo "Failed to auth to Vault. It may still be in the process of booting. Will sleep for 10 seconds and try again."
+    log "$description failed. Will sleep for 10 seconds and try again."
     sleep 10
   done;
 
-  echo "Failed to authenticate to Vault."
+  log "$description failed after 30 attempts."
   exit $exit_status
 }
 
@@ -47,7 +57,13 @@ data=$(cat <<EOF
 }
 EOF
 )
-retry_login "$data" "https://vault.service.consul:8200/v1/auth/aws/login"
+
+# run-consul is running on the background, so we have to wait for it and
+# we also have to for wait for vault server to be booted and unsealed before it can accept this request
+# so in case this fails we retry.
+login_output=$(retry \
+  "curl --fail --request POST --data '$data' https://vault.service.consul:8200/v1/auth/aws/login" \
+  "Trying to login to vault")
 
 # It is important to note that the default behavior is TOFU(trust on first use)
 # So if the pkcs7 certificate gets compromised, attempts to login again will be
@@ -107,10 +123,9 @@ retry_login "$data" "https://vault.service.consul:8200/v1/auth/aws/login"
 token=$(echo $login_output | jq -r .auth.client_token)
 
 # And use the token to perform operations on vault such as reading a secret
-response=$(curl \
-  -H "X-Vault-Token: $token" \
-  -X GET \
-  https://vault.service.consul:8200/v1/secret/example_gruntwork)
+response=$(retry \
+  "curl --fail -H 'X-Vault-Token: $token' -X GET https://vault.service.consul:8200/v1/secret/example_gruntwork" \
+  "Trying to read secret from vault")
 
 # If vault cli is installed we can also perform these operations with vault cli
 # The necessary environment variables have to be set

examples/vault-ec2-auth/user-data-vault.sh

@@ -18,27 +18,43 @@ readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem"
 /opt/consul/bin/run-consul --client --cluster-tag-key "${consul_cluster_tag_key}" --cluster-tag-value "${consul_cluster_tag_value}"
 /opt/vault/bin/run-vault --tls-cert-file "$VAULT_TLS_CERT_FILE"  --tls-key-file "$VAULT_TLS_KEY_FILE"
 
-# Initializes a vault server
-# run-vault is running on the background and we have to wait for it to be done,
-# so in case this fails we retry.
-function retry_init {
-  for i in $(seq 1 20); do
-    echo "Initializing Vault agent..."
+# Log the given message. All logs are written to stderr with a timestamp.
+function log {
+ local -r message="$1"
+ local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+ >&2 echo -e "$timestamp $message"
+}
+
+# A retry function that attempts to run a command a number of times and returns the output
+function retry {
+  local -r cmd="$1"
+  local -r description="$2"
+
+  for i in $(seq 1 30); do
+    log "$description"
+
     # The boolean operations with the exit status are there to temporarily circumvent the "set -e" at the
     # beginning of this script which exits the script immediatelly for error status while not losing the exit status code
-    server_output=$(/opt/vault/bin/vault operator init) && exit_status=0 || exit_status=$?
+    output=$(eval "$cmd") && exit_status=0 || exit_status=$?
+    log "$output"
     if [[ $exit_status -eq 0 ]]; then
+      echo "$output"
       return
     fi
-    echo "Failed to auth initialize Vault. Will sleep for 5 seconds and try again."
-    sleep 5
-  done
+    log "$description failed. Will sleep for 10 seconds and try again."
+    sleep 10
+  done;
 
-  echo "Failed to initialize Vault."
+  log "$description failed after 30 attempts."
   exit $exit_status
 }
 
-retry_init
+# Initializes a vault server
+# run-vault is running on the background and we have to wait for it to be done,
+# so in case this fails we retry.
+server_output=$(retry \
+  "/opt/vault/bin/vault operator init" \
+  "Trying to initialize vault")
 
 # The expected output should be similar to this:
 # ==========================================================================
@@ -74,7 +90,10 @@ echo "$server_output" | head -n 3 | awk '{ print $4; }' | xargs -l /opt/vault/bi
 export VAULT_TOKEN=$(echo "$server_output" | head -n 7 | tail -n 1 | awk '{ print $4; }')
 
 # Enables AWS authentication
-/opt/vault/bin/vault auth enable aws
+# This is an http request, and sometimes fails, hence we retry
+retry \
+  "/opt/vault/bin/vault auth enable aws" \
+  "Trying to enable aws auth"
 
 # Creates a policy that allows writing and reading from an "example_" prefix at "secret" backend
 /opt/vault/bin/vault policy write "example-policy" -<<EOF

examples/vault-iam-auth/outputs.tf

@@ -2,6 +2,10 @@ output "auth_client_public_ip" {
   value = "${aws_instance.example_auth_to_vault.public_ip}"
 }
 
+output "auth_client_instance_id" {
+  value = "${aws_instance.example_auth_to_vault.id}"
+}
+
 output "auth_role_arn" {
   value = "${aws_iam_role.example_instance_role.arn}"
 }

examples/vault-iam-auth/user-data-auth-client.sh

@@ -12,25 +12,29 @@ set -e
 # From: https://alestic.com/2010/12/ec2-user-data-output/
 exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 
-# Log the given message at the given level. All logs are written to stderr with a timestamp.
+# Log the given message. All logs are written to stderr with a timestamp.
 function log {
- local readonly message="$1"
- local readonly timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+ local -r message="$1"
+ local -r timestamp=$(date +"%Y-%m-%d %H:%M:%S")
  >&2 echo -e "$timestamp $message"
 }
 
 # A retry function that attempts to run a command a number of times and returns the output
 function retry {
-  local readonly cmd=$1
-  local readonly description=$2
+  local -r cmd="$1"
+  local -r description="$2"
 
   for i in $(seq 1 30); do
     log "$description"
 
     # The boolean operations with the exit status are there to temporarily circumvent the "set -e" at the
     # beginning of this script which exits the script immediatelly for error status while not losing the exit status code
     output=$(eval "$cmd") && exit_status=0 || exit_status=$?
-    if [[ $exit_status -eq 0 ]]; then
+    errors=$(echo "$output") | grep '^{' | jq -r .errors
+
+    log "$output"
+
+    if [[ $exit_status -eq 0 && -n "$output" && -z "$errors" ]]; then
       echo "$output"
       return
     fi
@@ -74,7 +78,7 @@ EOF
 # Retry in case the vault server is still booting and unsealing
 # Or in case run-consul running on the background didn't finish yet
 login_output=$(retry \
-  "curl --request POST --data '$data' https://vault.service.consul:8200/v1/auth/aws/login" \
+  "curl --fail --request POST --data '$data' https://vault.service.consul:8200/v1/auth/aws/login" \
   "Trying to login to vault")
 
 
@@ -90,10 +94,10 @@ login_output=$(retry \
 token=$(echo $login_output | jq -r .auth.client_token)
 
 # And use the token to perform operations on vault such as reading a secret
-response=$(curl \
-  -H "X-Vault-Token: $token" \
-  -X GET \
-  https://vault.service.consul:8200/v1/secret/example_gruntwork)
+# These is being retried because race conditions were causing this to come up null sometimes
+response=$(retry \
+  "curl --fail -H 'X-Vault-Token: $token' -X GET https://vault.service.consul:8200/v1/secret/example_gruntwork" \
+  "Trying to read secret from vault")
 
 # Vault cli alternative:
 # export VAULT_TOKEN=$token

examples/vault-iam-auth/user-data-vault.sh

@@ -18,27 +18,43 @@ readonly VAULT_TLS_KEY_FILE="/opt/vault/tls/vault.key.pem"
 /opt/consul/bin/run-consul --client --cluster-tag-key "${consul_cluster_tag_key}" --cluster-tag-value "${consul_cluster_tag_value}"
 /opt/vault/bin/run-vault --tls-cert-file "$VAULT_TLS_CERT_FILE"  --tls-key-file "$VAULT_TLS_KEY_FILE"
 
-# Initializes a vault server
-# run-vault is running on the background and we have to wait for it to be done,
-# so in case this fails we retry.
-function retry_init {
-  for i in $(seq 1 20); do
-    echo "Initializing Vault agent..."
+# Log the given message. All logs are written to stderr with a timestamp.
+function log {
+ local -r message="$1"
+ local readonly timestamp=$(date +"%Y-%m-%d %H:%M:%S")
+ >&2 echo -e "$timestamp $message"
+}
+
+# A retry function that attempts to run a command a number of times and returns the output
+function retry {
+  local -r cmd="$1"
+  local -r description="$2"
+
+  for i in $(seq 1 30); do
+    log "$description"
+
     # The boolean operations with the exit status are there to temporarily circumvent the "set -e" at the
-    # beginning of this script which exits the script immediatelly for error status, while not losing the exit status code
-    server_output=$(/opt/vault/bin/vault operator init) && exit_status=0 || exit_status=$?
+    # beginning of this script which exits the script immediatelly for error status while not losing the exit status code
+    output=$(eval "$cmd") && exit_status=0 || exit_status=$?
+    log "$output"
     if [[ $exit_status -eq 0 ]]; then
+      echo "$output"
       return
     fi
-    echo "Failed to auth initialize Vault. Will sleep for 5 seconds and try again."
-    sleep 5
-  done
+    log "$description failed. Will sleep for 10 seconds and try again."
+    sleep 10
+  done;
 
-  echo "Failed to initialize Vault."
+  log "$description failed after 30 attempts."
   exit $exit_status
 }
 
-retry_init
+# Initializes a vault server
+# run-vault is running on the background and we have to wait for it to be done,
+# so in case this fails we retry.
+server_output=$(retry \
+  "/opt/vault/bin/vault operator init" \
+  "Trying to initialize vault")
 
 # The expected output should be similar to this:
 # ==========================================================================
@@ -79,7 +95,10 @@ export VAULT_TOKEN=$(echo "$server_output" | head -n 7 | tail -n 1 | awk '{ prin
 # ==========================================================================
 
 # Enables AWS authentication
-/opt/vault/bin/vault auth enable aws
+# This is an http request, and sometimes fails, hence we retry
+retry \
+  "/opt/vault/bin/vault auth enable aws" \
+  "Trying to enable aws auth"
 
 # Creates a policy that allows writing and reading from an "example_" prefix at "secret" backend
 /opt/vault/bin/vault policy write "example-policy" -<<EOF

examples/vault-iam-auth/variables.tf

@@ -66,7 +66,7 @@ variable "vault_instance_type" {
 
 variable "consul_instance_type" {
   description = "The type of EC2 Instance to run in the Consul ASG"
-  default     = "t2.nano"
+  default     = "t2.micro"
 }
 
 variable "consul_cluster_tag_key" {

examples/vault-s3-backend/variables.tf

@@ -52,7 +52,7 @@ variable "vault_instance_type" {
 
 variable "consul_instance_type" {
   description = "The type of EC2 Instance to run in the Consul ASG"
-  default     = "t2.nano"
+  default     = "t2.micro"
 }
 
 variable "consul_cluster_tag_key" {