Merge pull request #8 from buildkite-plugins/update-vault-login

Update vault login to support AppRole authentication

Makefile

@@ -5,8 +5,8 @@ BUILDKITE_PIPELINE_SLUG=my_pipeline
 DUMP_ENV ?= true
 VAULT_DEV_ROOT_TOKEN_ID ?= 88F4384B-98E9-4AE3-B00C-F55678F89080
 
-TESTER_VAULT_VERSION ?= 0.9.1
-SVC_VAULT_VERSION ?= 0.9.1
+TESTER_VAULT_VERSION ?= 1.11.2
+SVC_VAULT_VERSION ?= 1.11.2
 
 all:;: '$(VAULT_ADDR)' \
 	'$(TESTER_VAULT_VERSION)' \

README.md

@@ -13,20 +13,26 @@ Different types of secrets are supported and exposed to your builds in appropria
 - Environment Variables for strings
 - `git-credential` via git's credential.helper
 
-## Example
+## ENV example
 
-The following pipeline downloads a private key from `https://my-vault-server/data/buildkite/{pipeline}/ssh_private_key` and set of environment variables from `https://my-vault-server/data/buildkite/{pipeline}/environment`.
+The following pipeline downloads env secrets stored in `https://my-vault-server/secret/buildkite/{pipeline}/env` and git-credentials from `https://my-vault-server/secret/buildkite/{pipeline}/git-credentials`
 
-The private key is exposed to both the checkout and the command as an ssh-agent instance. The secrets in the env file are exposed as environment variables.
+The keys in the `env` secret are exposed in the `checkout` and `command` as environment variables. The git-credentials are exposed as an environment variable `GIT_CONFIG_PARAMETERS` and are also exposed in the `checkout` and `command`.
 
 ```yml
 steps:
   - command: ./run_build.sh
     plugins:
-      - mikeknox/vault-secrets#v0.1.:
-          server: my-vault-server
+      - buildkite-plugins/vault-secrets#v0.2.0:
+          server: "https://my-vault-server"
+          path: secret/buildkite
+          auth:
+            method: approle
+            role-id: "my-role-id"
+            secret-env: "VAULT_SECRET_ID"
 ```
 
+
 ## Uploading Secrets
 
 Secrets are uploading using the Vault CLI, as a `base64` encoded blob in a field called *value*.

hooks/environment

@@ -24,17 +24,13 @@ processSshSecrets() {
 
 processEnvSecrets() {
   local vaultServer="$1"
-  local envKey="$2"
+  local key="$2"
   local envscript=''
-
-  echo "Checking vault key ${envKey}" >&2
-  for key in $(list_secrets "$vaultServer" "$envKey")  ; do
-    echo "Downloading env secret from ${key}" >&2;
+  echo "Downloading env secret from ${key}" >&2;
     if ! envscript=$(echo "${envscript:-}" && secret_download "${vaultServer}" "${key}") ; then
       echo "+++ :warning: Failed to download env from $key" >&2;
       exit 1
-    fi
-  done
+   fi
   echo "Evaluating ${#envscript} bytes of env"
   set -o allexport
   eval "$envscript"
@@ -69,7 +65,7 @@ _source="${BASH_SOURCE[0]}"
 [ -z "${_source:-}" ] && _source="${0}"
 basedir="$( cd "$( dirname "${_source}" )" && cd .. && pwd )"
 
-# shellcheck disable=SC1090
+# shellcheck source=lib/shared.bash
 . "$basedir/lib/shared.bash"
 
 TMPDIR=${TMPDIR:-/tmp}
@@ -90,7 +86,6 @@ if [[ -n "$vault_server" ]] ; then
     "${vault_path}"
     "${VAULT_BASE_PATH}"
   )
-
   secrets=''
   for key in ${secret_paths[*]} ; do
     echo "Checking vault secrets ${key}" >&2
@@ -101,7 +96,6 @@ if [[ -n "$vault_server" ]] ; then
     # shellcheck disable=SC2034
     _key_base="$(dirname "$secret")"
     _key_name="$(basename "$secret")"
-
     case "${_key_name:-}" in
       env|environment) processEnvSecrets "$vault_server" "$secret" ;;
       private_ssh_key|id_rsa_github) processSshSecrets "$vault_server" "$secret" ;;

lib/shared.bash

@@ -11,25 +11,40 @@ esac
 
 vault_auth() {
   local server="$1"
-  local auth_params=''
-  # role is defined in Vault Configuration
-  # -header=<foo> # used to set X-Auth
-  # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD - aws
-  # BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_HEADER
-  # BUILDKITE_PLUGIN_VAULT_SECRETS_ROLE
-  [ -n "${server:-}" ] && auth_params="${auth_params} -address=${server}"
-  [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD:-}" ] && auth_params="${auth_params} -method=${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD}"
-  [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_HEADER:-}" ] && auth_params="${auth_params} -header_value=${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_HEADER}"
-  [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_ROLE:-}" ] && auth_params="${auth_params} role=${BUILDKITE_PLUGIN_VAULT_SECRETS_ROLE}"
-
-  if [ -n "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD:-}" ] ; then
-    # don't output the token to log, even though it's a temporary token
-    # shellcheck disable=SC2086
-    vault auth $auth_params | grep -v ^token:
+
+  # Currently we only support AppRole authentication.
+  # These values are referenced when authenticating to the Vault server:
+  #   BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD - approle
+
+  #   RoleID and SecretID should be stored securely on the agent, there are probably better ways to do this, but here is a start
+  #   We'll use these two values for the RoleID and SecretID:
+  #     BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_SECRET_ID
+  #     BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_ROLE_ID
+  #
+  #   For now, you will need to define the secret ID oustide of the plugin, though this will probably change.
+
+   # approle authentication
+  if [ "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_METHOD:-}" = "approle" ]; then
+
+    secret_var="${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_SECRET_ENV:-$VAULT_SECRET_ID}"
+
+    if [[ -z "${!secret_var:-}" ]]; then
+      echo "+++  🚨 No vault secret id found in \$${secret_var}"
+      exit 1
+    fi
+
+    # export the vault token to be used for this job - this command writes to the auth/approle/login endpoint
+    # on success, vault will return the token which we export as VAULT_TOKEN for this shell
+    export VAULT_TOKEN
+    if ! VAULT_TOKEN=$(vault write -field=token -address="$server" auth/approle/login \
+     role_id="$BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_ROLE_ID" \
+     secret_id="${!secret_var:-}"); then
+      echo "Failed to get vault token"
+    fi
+
+    echo "Successfully authenticated with RoleID ${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_ROLE_ID} and updated vault token"
+
     return "${PIPESTATUS[0]}"
-  else
-    # shellcheck disable=SC2086
-    echo "${BUILDKITE_PLUGIN_VAULT_SECRETS_AUTH_TOKEN:-}" | vault auth ${auth_params:-} -
   fi
 }
 
@@ -38,7 +53,11 @@ list_secrets() {
   local key="$2"
 
   local _list
-  _list=$(vault list -address="$server" -format=yaml "$key" | sed 's/^- //g' )
+
+  if ! _list=$(vault kv list -address="$server" -format=yaml "$key" | sed 's/^- //g'); then
+    echo "unable to list secrets" >&2;
+    return "${PIPESTATUS[0]}"
+  fi
   local retVal=${PIPESTATUS[0]}
 
   for lineItem in ${_list} ; do
@@ -57,7 +76,7 @@ secret_exists() {
   local _key_name
   _key_name="$(basename "$key")"
   local _list
-  _list=$(vault list -address="$server" -format=yaml "$_key_base" )
+  _list=$(vault kv list -address="$server" -format=yaml "$_key_base" )
 
   echo "${_list}" | grep "^- ${_key_name}$" >& /dev/null
   # shellcheck disable=SC2181
@@ -71,8 +90,10 @@ secret_exists() {
 secret_download() {
   local server="$1"
   local key="$2"
-
-  _secret=$(vault read -address="${server}" -field=value "$key" | base64 $BASE64_DECODE_ARGS)
+  if ! _secret=$(vault kv get -field=data -format=yaml secret/buildkite/env | sed 's/: /=/g' ); then
+    echo "Failed to download secrets"
+    exit 1
+  fi
   # shellcheck disable=SC2181
   if [ "$?" -ne 0 ] ; then
     return 1
@@ -89,6 +110,7 @@ add_ssh_private_key_to_agent() {
   fi
 
   echo "Loading ssh-key into ssh-agent (pid ${SSH_AGENT_PID:-})" >&2;
+
   echo "$ssh_key" | env SSH_ASKPASS="/bin/false" ssh-add -
 }
 

plugin.yml

@@ -1,6 +1,6 @@
 name: "Vault Secrets"
 description: "Expose build secrets stored in Vault to your jobs"
-author: "@mikeknox"
+author: "@mikeknox, @buildkite-plugins"
 public: true
 requirements:
   - curl
@@ -10,7 +10,19 @@ configuration:
   properties:
     server:
       type: string
+    auth:
+      type: object
+      properties:
+        method:
+          enum:
+            - 'approle'
+            - ''
+        role-id:
+          type: string
   required:
     - server
+    - auth
   additionalProperties: false
+  dependencies:
+    auth: [ method ]