Skip to content

Commit

Permalink
Merge pull request #29 from staticfloat/sf/group_capable
Browse files Browse the repository at this point in the history 
Sf/group capable
  • Loading branch information
@staticfloat
staticfloat authored Jun 10, 2024
2 parents 0d311d5 + ff817f9 commit 78ec7a7
Show file tree
Hide file tree
Showing 8 changed files with 98 additions and 68 deletions.
2 changes: 1 addition & 1 deletion bin/create_repo_key
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash

## This script creates a "repository key". This key is stored, encrypted, within a special folder
## that the agent knows to look inside of when decrypting secrets. It also sets the repository up
Expand Down
2 changes: 1 addition & 1 deletion bin/sign_treehashes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,4 +44,4 @@ for YAML_PATH in ${YAML_PATHS[@]}; do
EOD
fi
done
done
done
1 change: 0 additions & 1 deletion bin/verify_treehashes
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ for YAML_PATH in ${YAML_PATHS[@]}; do
PIPELINE_TREEHASH_FILESOURCE="$(cut -d'&' -f4 <<<"${TRIPLET}" | tr -d '"')"

# Compare decrypted treehash with calculated treehash
vecho "About to decrypt ${PIPELINE_ENCRYPTED_TREEHASH} with ${REPO_KEY_PATH}"
PIPELINE_DECRYPTED_TREEHASH="$(base64dec <<<"${PIPELINE_ENCRYPTED_TREEHASH}" | decrypt_aes "${REPO_KEY_PATH}" 2>/dev/null || true)"
if [[ "${PIPELINE_DECRYPTED_TREEHASH}" == "${PIPELINE_TREEHASH}" ]]; then
echo "[${YAML_PATH}] -> ${PIPELINE_PATH}: ✔️"
Expand Down
10 changes: 7 additions & 3 deletions hooks/post-command
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,18 @@ for PIPELINE_IDX in "${!SIGNED_PIPELINES[@]}"; do

# Hash up the inputs
readarray -d '' -t PIPELINE_INPUTS < <(collect_buildkite_array "BUILDKITE_PLUGIN_CRYPTIC_SIGNED_PIPELINES_${PIPELINE_IDX}_INPUTS")
vecho " -> Performing pipeline launch:"
vecho " -> ${PIPELINE_PATH}"
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )
for PATTERN in "${PIPELINE_INPUTS[@]}"; do
INPUT_TREEHASHES+=( "$(collect_glob_pattern "${PATTERN}" | calc_treehash)" )
HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)"
vecho " + ${HASH} <- ${PATTERN}"
INPUT_TREEHASHES+=( "${HASH}" )
done

# Hash all treehashes together to get full input hash
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
vecho " ∟ ${FULL_TREEHASH}"

# Verify this with the treehash signature
SIGNATURE_VAR="BUILDKITE_PLUGIN_CRYPTIC_SIGNED_PIPELINES_${PIPELINE_IDX}_SIGNATURE"
Expand All @@ -75,7 +80,6 @@ for PIPELINE_IDX in "${!SIGNED_PIPELINES[@]}"; do
fi
if [[ "$(decrypt_aes "${UNENCRYPTED_REPO_KEY_PATH}" <"${SIGNATURE_FILE}")" != "${FULL_TREEHASH}" ]]; then
SIGNATURE_FAIL_MSG="Pipeline '${PIPELINE_PATH}' fails treehash signature check! You may need to re-run cryptic/bin/sign_treehashes!"
echo "${SIGNATURE_FAIL_MSG}" >&2

HASH_OVERRIDE_VAR="BUILDKITE_PLUGIN_CRYPTIC_SIGNED_PIPELINES_${PIPELINE_IDX}_ALLOW_HASH_OVERRIDE"
if [[ -v "${HASH_OVERRIDE_VAR}" ]] && [[ "${!HASH_OVERRIDE_VAR}" == "true" ]]; then
Expand Down Expand Up @@ -103,7 +107,7 @@ for PIPELINE_IDX in "${!SIGNED_PIPELINES[@]}"; do
else
# Execute `die` in a subshell so that we can print out failure messages for each pipeline,
# then fail out once at the end, ignoring the `exit` and failure that it creates.
(die "Refusing to continue execution; pipeline '${PIPELINE_PATH}' fails treehash signature check! You may need to re-run cryptic/bin/sign_treehashes!"; ) || true
(die "${SIGNATURE_FAIL_MSG}"; ) || true
SHOULD_FAIL=true
continue
fi
Expand Down
4 changes: 2 additions & 2 deletions lib/argparse.sh
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash

VERBOSE="${VERBOSE:-false}"
function verbose() {
Expand Down Expand Up @@ -95,4 +95,4 @@ if verbose; then
fi

# Restore positional parameters back to `$1`, `$2`, etc...
set -- "${POSITIONAL[@]}"
set -- "${POSITIONAL[@]}"
8 changes: 6 additions & 2 deletions lib/common.sh
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#!/usr/bin/env bash

set -eou pipefail
shopt -s extglob
Expand All @@ -20,7 +20,7 @@ function die() {
}

# Returns true if verbose mode is enabled
VERBOSE="${VERBOSE:-false}"
VERBOSE="${BUILDKITE_PLUGIN_CRYPTIC_VERBOSE:-${VERBOSE:-false}}"
function verbose() {
[[ "${VERBOSE}" == "true" ]]
}
Expand Down Expand Up @@ -439,6 +439,10 @@ function find_repository_root() {
}

function find_repo_key() {
if [[ -v "REPO_KEY_PATH" ]]; then
return
fi

# First, check to see if we have a decrypted repo key:
REPO_KEY_PATH="${REPO_ROOT}/.buildkite/cryptic_repo_keys/repo_key"
if [[ -f "${REPO_KEY_PATH}" ]]; then
Expand Down
135 changes: 77 additions & 58 deletions lib/yaml_extraction_prologue.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,20 @@ fi
# Extract the `variables:` section of a cryptic `pipeline.yml` plugin section
function extract_encrypted_variables() {
# Iterate over the steps in the yaml file
(shyaml get-values-0 steps <"${1}" || true) |
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
# For each step, get its list of plugins
(shyaml get-values-0 plugins <<<"${STEP}" 2>/dev/null || true) |
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml keys-0 <<<"${PLUGINS}" || true) |
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# For each plugin, if its `cryptic`, extract the variables
(shyaml get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" 2>/dev/null || true) |
(shyaml -q get-values-0 "${PLUGIN_NAME}.variables" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' VAR; do
printf "%s\n" "${VAR}"
done
Expand All @@ -34,20 +34,20 @@ function extract_encrypted_variables() {
# Extract all variables that match "CRYPTIC_ADHOC_SECRET_*"
function extract_adhoc_encrypted_variables() {
# Iterate over any global env mappings
(shyaml keys-0 env <"${1}" 2>/dev/null || true) |
(shyaml -q keys-0 env <"${1}" || true) |
while IFS='' read -r -d '' VARNAME; do
if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then
printf "%s\n" "${VARNAME:21}=$(shyaml get-value env.${VARNAME} <"${1}")"
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <"${1}")"
fi
done

# Iterate over the steps in the yaml file
(shyaml get-values-0 steps <"${1}" || true) |
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
(shyaml keys-0 env <<<"${STEP}" 2>/dev/null || true) |
(shyaml -q keys-0 env <<<"${STEP}" || true) |
while IFS='' read -r -d '' VARNAME; do
if [[ "${VARNAME}" == CRYPTIC_ADHOC_SECRET_* ]]; then
printf "%s\n" "${VARNAME:21}=$(shyaml get-value env.${VARNAME} <<<"${STEP}")"
printf "%s\n" "${VARNAME:21}=$(shyaml -q get-value env.${VARNAME} <<<"${STEP}")"
fi
done
done
Expand All @@ -56,20 +56,20 @@ function extract_adhoc_encrypted_variables() {
# Extract the `files:` section of a cryptic `pipeline.yml` plugin section
function extract_encrypted_files() {
# Iterate over the steps in the yaml file
(shyaml get-values-0 steps <"${1}" || true) |
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
# For each step, get its list of plugins
(shyaml get-values-0 plugins <<<"${STEP}" 2>/dev/null || true) |
(shyaml -q get-values-0 plugins <<<"${STEP}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml keys-0 <<<"${PLUGINS}" || true) |
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# For each plugin, if its `cryptic`, extract the files
(shyaml get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" 2>/dev/null || true) |
(shyaml -q get-values-0 "${PLUGIN_NAME}.files" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' FILE; do
FILE="$(echo ${FILE} | tr -d '"')"
printf "%s\n" "${FILE}"
Expand All @@ -88,56 +88,75 @@ function extract_pipeline_treehashes() {
vecho "Extracting treehashes from '${YAML_PATH}'"

# Iterate over the steps in the yaml file
(shyaml get-values-0 steps <"${1}" || true) |
(shyaml -q get-values-0 steps <"${1}" || true) |
while IFS='' read -r -d '' STEP; do
# For each step, get its list of plugins
(shyaml get-values-0 plugins <<<"${STEP}" 2>/dev/null || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi
# If this step is a `group` step, let's iterate over each of its steps
if shyaml -q get-value 'group' >/dev/null <<<"${STEP}"; then
(shyaml -q get-values-0 steps <<<"${STEP}" || true) |
while IFS='' read -r -d '' INNER_STEP; do
extract_plugin_treehashes "${INNER_STEP}"
done
else
extract_plugin_treehashes "${STEP}"
fi
done

# For each plugin, if its `cryptic`, walk over the pipelines
(shyaml get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" 2>/dev/null || true) |
while IFS='' read -r -d '' PIPELINE; do
# For each signed pipeline, get its pipeline path and its inputs
PIPELINE_PATH="$(shyaml get-value "pipeline" <<<"${PIPELINE}" 2>/dev/null || true)"

# Start by calculating the treehash of the yaml file
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )

# Next, calculate the treehash of the rest of the glob patterns
for PATTERN in $(shyaml get-values "inputs" <<<"${PIPELINE}" 2>/dev/null || true); do
INPUT_TREEHASHES+=( "$(collect_glob_pattern "${PATTERN}" | calc_treehash)" )
done

# Calculate full treehash
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"

# If `signature_file` is defined, use it!
local BASE64_ENCRYPTED_TREEHASH=""
local TREEHASH_FILE_SOURCE=""
if shyaml get-value "signature_file" <<<"${PIPELINE}" 2>/dev/null >/dev/null; then
TREEHASH_FILE_SOURCE="$(shyaml get-value "signature_file" <<<"${PIPELINE}" 2>/dev/null)"
if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then
BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")"
fi
else
# Try to extract the signature from the yaml directly too
BASE64_ENCRYPTED_TREEHASH="$(shyaml get-value "signature" <<<"${PIPELINE}" 2>/dev/null || true)"
fi
# Don't stay in `${REPO_ROOT}`
popd >/dev/null
}

function extract_plugin_treehashes() {
# Get the list of plugins
(shyaml -q get-values-0 plugins <<<"${1}" || true) |
while IFS='' read -r -d '' PLUGINS; do
# Get the plugin names
(shyaml -q keys-0 <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PLUGIN_NAME; do
# Skip plugins that are not named `cryptic`
if [[ "${PLUGIN_NAME}" != staticfloat/cryptic* ]]; then
continue
fi

# For each plugin, if its `cryptic`, walk over the pipelines
(shyaml -q get-values-0 "${PLUGIN_NAME}.signed_pipelines" <<<"${PLUGINS}" || true) |
while IFS='' read -r -d '' PIPELINE; do
# For each signed pipeline, get its pipeline path and its inputs
PIPELINE_PATH="$(shyaml -q get-value "pipeline" <<<"${PIPELINE}" || true)"

# Print out treehash and pipeline path
printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}"
vecho " -> Found pipeline launch:"
vecho " -> ${PIPELINE_PATH}"

# Start by calculating the treehash of the yaml file
INPUT_TREEHASHES=( "$(calc_treehash <<<"${PIPELINE_PATH}")" )

# Next, calculate the treehash of the rest of the glob patterns
readarray -d '' PATTERNS -t < <(shyaml -q get-values-0 "inputs" <<<"${PIPELINE}")
for PATTERN in "${PATTERNS[@]}"; do
HASH="$(collect_glob_pattern "${PATTERN}" | calc_treehash)"
vecho " + ${HASH} <- ${PATTERN}"
INPUT_TREEHASHES+=( "${HASH}" )
done

# Calculate full treehash
FULL_TREEHASH="$(printf "%s" "${INPUT_TREEHASHES[@]}" | calc_shasum)"
vecho " ∟ ${FULL_TREEHASH}"

# If `signature_file` is defined, use it!
local BASE64_ENCRYPTED_TREEHASH=""
local TREEHASH_FILE_SOURCE=""
if shyaml get-value "signature_file" <<<"${PIPELINE}" >/dev/null; then
TREEHASH_FILE_SOURCE="$(shyaml -q get-value "signature_file" <<<"${PIPELINE}")"
if [[ -f "${TREEHASH_FILE_SOURCE}" ]]; then
BASE64_ENCRYPTED_TREEHASH="$(base64enc <"${TREEHASH_FILE_SOURCE}")"
fi
else
# Try to extract the signature from the yaml directly too
BASE64_ENCRYPTED_TREEHASH="$(shyaml -q get-value "signature" <<<"${PIPELINE}" || true)"
fi

# Print out treehash and pipeline path
printf "%s&%s&%s&%s\n" "${PIPELINE_PATH}" "${FULL_TREEHASH}" "${BASE64_ENCRYPTED_TREEHASH}" "${TREEHASH_FILE_SOURCE}"
done
done
done

# Don't stay in `${REPO_ROOT}`
popd >/dev/null
}
4 changes: 4 additions & 0 deletions plugin.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,8 @@ configuration:
unsigned_pipelines:
type: array

# Set this to allow for greater verbosity in the cryptic plugin's output
verbose:
type: bool

additionalProperties: false

0 comments on commit 78ec7a7

Please sign in to comment.