diff --git a/README.md b/README.md index e2cf697..4269378 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,12 @@ This is an example of a daemon process that monitors a filesystem mountpoint and ## Assumptions: 1. Code is running on an AWS EC2 instance -2. The instance and AMI use HVM virtualization -3. The instance AMI allows device names like `/dev/xvdb*` and will not remap them -4. The instance is using a Linux based OS with either **upstart** or **systemd** system initialization -5. The instance has a IAM Instance Profile with appropriate permissions to create and attach new EBS volumes. See the [IAM Instance Profile](#a-note-on-the-iam-instance-profile) section below for more details -6. That prerequisites are installed on the instance: +2. The instance has awscli v2 installed +3. The instance and AMI use HVM virtualization +4. The instance AMI allows device names like `/dev/xvdb*` and will not remap them +5. The instance is using a Linux based OS with either **upstart** or **systemd** system initialization +6. The instance has a IAM Instance Profile with appropriate permissions to create and attach new EBS volumes. See the [IAM Instance Profile](#a-note-on-the-iam-instance-profile) section below for more details +7. That prerequisites are installed on the instance: 1. jq 2. btrfs-progs 3. lvm2 @@ -82,6 +83,9 @@ Options --volume-throughput VOLUMETHOUGHPUT Volume throughput for gp3 (default: 125) + --not-encrypted Flag to make the volume un-encyrpted. Default is to create + an encrypted volume + --min-ebs-volume-size SIZE_GB Mimimum size in GB of new volumes created by the instance. (Default: 150) diff --git a/bin/create-ebs-volume b/bin/create-ebs-volume index cf117be..1260616 100755 --- a/bin/create-ebs-volume +++ b/bin/create-ebs-volume @@ -35,7 +35,8 @@ export EBS_AUTOSCALE_CONFIG_FILE=/etc/ebs-autoscale.json IMDSV2=$(get_config_value .imdsv2) initialize -USAGE=$(cat < @@ -77,7 +78,7 @@ if [ "$#" -lt 1 ]; then fi function error() { - logthis "Error: $1" + logthis "Error: $1" "error" echo "Error: $1" >&2 exit 1 } @@ -92,55 +93,55 @@ MAX_CREATED_VOLUMES=$MAX_ATTACHED_VOLUMES # parse options PARAMS="" -while (( "$#" )); do +while (("$#")); do case "$1" in - -s|--size) - SIZE=$2 - shift 2 - ;; - -t|--type) - TYPE=$2 - shift 2 - ;; - -i|--iops) - IOPS=$2 - shift 2 - ;; - --throughput) - THROUGHPUT=$2 - shift 2 - ;; - --not-encrypted) - unset ENCRYPTED - shift - ;; - --max-total-created-size) - MAX_LOGICAL_VOLUME_SIZE=$2 - shift 2 - ;; - --max-attached-volumes) - MAX_ATTACHED_VOLUMES=$2 - shift 2 - ;; - --max-created-volumes) - MAX_CREATED_VOLUMES=$2 - shift 2 - ;; - -v|--verbose) - VERBOSE=1 - shift - ;; - --) # end parsing - shift - break - ;; - -*|--*=) - error "unsupported argument $1" - ;; - *) # positional arguments - PARAMS="$PARAMS $1" - shift - ;; + -s | --size) + SIZE=$2 + shift 2 + ;; + -t | --type) + TYPE=$2 + shift 2 + ;; + -i | --iops) + IOPS=$2 + shift 2 + ;; + --throughput) + THROUGHPUT=$2 + shift 2 + ;; + --not-encrypted) + unset ENCRYPTED + shift + ;; + --max-total-created-size) + MAX_LOGICAL_VOLUME_SIZE=$2 + shift 2 + ;; + --max-attached-volumes) + MAX_ATTACHED_VOLUMES=$2 + shift 2 + ;; + --max-created-volumes) + MAX_CREATED_VOLUMES=$2 + shift 2 + ;; + -v | --verbose) + VERBOSE=1 + shift + ;; + --) # end parsing + shift + break + ;; + -* | --*=) + error "unsupported argument $1" + ;; + *) # positional arguments + PARAMS="$PARAMS $1" + shift + ;; esac done @@ -154,7 +155,7 @@ if [[ ! "$SIZE" ]]; then error "missing required argument --size" fi -alphabet=( {a..z} ) +alphabet=({a..z}) function get_next_logical_device() { for letter in ${alphabet[@]}; do @@ -177,32 +178,32 @@ function create_and_attach_volume() { # Output Example: # {Key=Name,Value=Jenkins},{Key=Owner,Value=DevOps} instance_tags=$( - aws ec2 describe-tags \ - --region $region \ - --filters "Name=resource-id,Values=$instance_id" | jq -r .Tags | jq -c 'map({Key, Value})' | tr -d '[]"' | sed 's/{Key:/{Key=/g ; s/,Value:/,Value=/g ; s/{Key=aws:[^}]*}//g ; s/,\{2,\}/,/g ; s/,$//g ; s/^,//g' - ) + aws ec2 describe-tags \ + --region $region \ + --filters "Name=resource-id,Values=$instance_id" | jq -r .Tags | jq -c 'map({Key, Value})' | tr -d '[]"' | sed 's/{Key:/{Key=/g ; s/,Value:/,Value=/g ; s/{Key=aws:[^}]*}//g ; s/,\{2,\}/,/g ; s/,$//g ; s/^,//g' + ) local max_attempts=10 local attached_volumes="" - for i in $(eval echo "{0..$max_attempts}") ; do + for i in $(eval echo "{0..$max_attempts}"); do attached_volumes=$( - aws ec2 describe-volumes \ - --region $region \ - --filters "Name=attachment.instance-id,Values=$instance_id" + aws ec2 describe-volumes \ + --region $region \ + --filters "Name=attachment.instance-id,Values=$instance_id" ) if [ $? -eq 0 ]; then - break + break elif [ $i -eq $max_attempts ]; then logthis "Could not determine the number of attached_volumes after $i attempts. Last response was: $attached_volumes" break fi - sleep $(( 2 ** i + $RANDOM % 3)) + sleep $((2 ** i + $RANDOM % 3)) done local created_volumes="" - for i in $(eval echo "{0..$max_attempts}") ; do + for i in $(eval echo "{0..$max_attempts}"); do created_volumes=$( aws ec2 describe-volumes \ --region $region \ @@ -210,16 +211,16 @@ function create_and_attach_volume() { ) if [ $? -eq 0 ]; then - break + break elif [ $i -eq $max_attempts ]; then logthis "Could not determine the number of created_volumes after $i attempts. Last response was: $created_volumes" break fi - sleep $(( 2 ** i + $RANDOM % 3)) + sleep $((2 ** i + $RANDOM % 3)) done local total_created_size="" - for i in $(eval echo "{0..$max_attempts}") ; do + for i in $(eval echo "{0..$max_attempts}"); do total_created_size=$( aws ec2 describe-volumes \ --region $region \ @@ -229,12 +230,12 @@ function create_and_attach_volume() { ) if [ $? -eq 0 ]; then - break + break elif [ $i -eq $max_attempts ]; then logthis "Could not determine the total_created_size after $i attempts. Last response was: $total_created_size" break fi - sleep $(( 2 ** i + $RANDOM % 3)) + sleep $((2 ** i + $RANDOM % 3)) done # check how much EBS storage this instance has created @@ -243,15 +244,15 @@ function create_and_attach_volume() { fi # check how many volumes this instance has created - if [ "`echo $created_volumes | jq '.Volumes | length'`" -ge "$MAX_CREATED_VOLUMES" ]; then + if [ "$(echo $created_volumes | jq '.Volumes | length')" -ge "$MAX_CREATED_VOLUMES" ]; then error "maximum number of created volumes reached ($MAX_CREATED_VOLUMES)" fi # check how many volumes are currently attached - if [ "`echo $attached_volumes | jq '.Volumes | length'`" -ge "$MAX_ATTACHED_VOLUMES" ]; then + if [ "$(echo $attached_volumes | jq '.Volumes | length')" -ge "$MAX_ATTACHED_VOLUMES" ]; then error "maximum number of attached volumes reached ($MAX_ATTACHED_VOLUMES)" fi - + # check if there are available device names local device=$(get_next_logical_device) if [ -z "$device" ]; then @@ -262,49 +263,52 @@ function create_and_attach_volume() { # create the volume local tmpfile=$(mktemp /tmp/ebs-autoscale.create-volume.XXXXXXXXXX) local volume_opts="--size $SIZE --volume-type $TYPE" - local IOPS_TYPES=( io1 io2 gp3 ) + local IOPS_TYPES=(io1 io2 gp3) if [[ " ${IOPS_TYPES[*]} " =~ " ${TYPE} " ]]; then volume_opts="$volume_opts --iops $IOPS"; fi if [ "$TYPE" == "gp3" ]; then volume_opts="$volume_opts --throughput $THROUGHPUT"; fi if [ "$ENCRYPTED" == "1" ]; then volume_opts="$volume_opts --encrypted"; fi - local timestamp=$(date "+%F %T UTC%z") # YYYY-mm-dd HH:MM:SS UTC+0000 + local timestamp=$(date "+%F %T UTC%z") # YYYY-mm-dd HH:MM:SS UTC+0000 local volume="" - for i in $(eval echo "{0..$max_attempts}") ; do - - # The $instance_tags variable could be empty and will cause a TagSpecifications[0].Tags[0] error if - # it is passed as an empty value because it must be comma-separated from the other key-value pairs. - # Use a Shell Parameter Expansion to determine if the variable contains a value or not. If it has a value, - # append a comma at the end so the aws cli syntax is compliant when it is subbed into the tag_specification variable. - local instance_tags=${instance_tags:+${instance_tags},} - local tag_specification="ResourceType=volume,Tags=[$instance_tags{Key=source-instance,Value=$instance_id},{Key=amazon-ebs-autoscale-creation-time,Value=$timestamp}]" - - # Note: Shellcheck says the $vars in this command should be double quoted to prevent globbing and word-splitting, - # but this ends up making the '--encrypted' argument to fail during the execution of the install script. Conversely, NOT putting double-quotes - # around $tag_specification causes a parsing error due to the space in the $timestamp value (added to $tag_specification above). - - local volume=$(\ - aws ec2 create-volume \ - --region $region \ - --availability-zone $availability_zone \ - $volume_opts \ - --tag-specification "$tag_specification" \ - 2> $tmpfile - ) - - if [ $? -eq 0 ]; then - break - elif [ $i -eq $max_attempts ]; then - logthis "Could not create a volume after $i attempts. Last response was: $volume" - break - fi - sleep $(( 2 ** i + $RANDOM % 3)) + for i in $(eval echo "{0..$max_attempts}"); do + + # The $instance_tags variable could be empty and will cause a TagSpecifications[0].Tags[0] error if + # it is passed as an empty value because it must be comma-separated from the other key-value pairs. + # Use a Shell Parameter Expansion to determine if the variable contains a value or not. If it has a value, + # append a comma at the end so the aws cli syntax is compliant when it is subbed into the tag_specification variable. + local instance_tags=${instance_tags:+${instance_tags},} + local tag_specification="ResourceType=volume,Tags=[$instance_tags{Key=source-instance,Value=$instance_id},{Key=amazon-ebs-autoscale-creation-time,Value=$timestamp}]" + + # Note: Shellcheck says the $vars in this command should be double quoted to prevent globbing and word-splitting, + # but this ends up making the '--encrypted' argument to fail during the execution of the install script. Conversely, NOT putting double-quotes + # around $tag_specification causes a parsing error due to the space in the $timestamp value (added to $tag_specification above). + + local volume=$( + aws ec2 create-volume \ + --region $region \ + --availability-zone $availability_zone \ + $volume_opts \ + --tag-specification "$tag_specification" \ + 2>$tmpfile + ) + + if [ $? -eq 0 ]; then + logthis "volume created successfully" + break + elif [ $i -eq $max_attempts ]; then + logthis "Could not create a volume after $i attempts. Last response was: $volume" "error" + break + else + logthis "Attempt $i: Could not create volume, will retry. Error: $volume" "error" + fi + sleep $((2 ** i + $RANDOM % 3)) done - local volume_id=`echo $volume | jq -r '.VolumeId'` + local volume_id=$(echo $volume | jq -r '.VolumeId') if [ -z "$volume_id" ]; then - logthis "$(cat $tmpfile)" # log captured error - cat $tmpfile # print captured error (e.g. when called during install) + logthis "$(cat $tmpfile)" # log captured error + cat $tmpfile # print captured error (e.g. when called during install) rm $tmpfile error "could not create volume" @@ -314,11 +318,11 @@ function create_and_attach_volume() { logthis "created volume: $volume_id [ $volume_opts ]" # In theory this shouldn't need to loop as aws ec2 wait will retry but I have seen it exceed request limits - for i in {1..3} ; do - if aws ec2 wait volume-available --region $region --volume-ids $volume_id; then - logthis "volume $volume_id available" - break - fi + for i in {1..3}; do + if aws ec2 wait volume-available --region $region --volume-ids $volume_id; then + logthis "volume $volume_id available" + break + fi done # Need to assure that the created volume is successfully attached to be @@ -332,16 +336,16 @@ function create_and_attach_volume() { --device $device \ --instance-id $instance_id \ --volume-id $volume_id \ - > /dev/null - + >/dev/null + status="$?" if [ ! "$status" -eq 0 ]; then logthis "deleting volume $volume_id" aws ec2 delete-volume \ --region $region \ --volume-id $volume_id \ - > /dev/null - + >/dev/null + error "could not attach volume to instance" fi set -e @@ -360,7 +364,7 @@ function create_and_attach_volume() { --region $region \ --instance-id $instance_id \ --block-device-mappings "DeviceName=$device,Ebs={DeleteOnTermination=true,VolumeId=$volume_id}" \ - > /dev/null + >/dev/null logthis "volume $volume_id DeleteOnTermination ENABLED" echo $device diff --git a/config/ebs-autoscale.json b/config/ebs-autoscale.json index 372478f..852bd79 100644 --- a/config/ebs-autoscale.json +++ b/config/ebs-autoscale.json @@ -10,7 +10,7 @@ "type": "%%VOLUMETYPE%%", "iops": "%%VOLUMEIOPS%%", "throughput": "%%VOLUMETHOUGHPUT%%", - "encrypted": 1 + "encrypted": "%%ENCRYPTED%%" }, "detection_interval": 2, "limits": { diff --git a/install.sh b/install.sh index 89c0ad0..a1f025e 100644 --- a/install.sh +++ b/install.sh @@ -62,6 +62,9 @@ Options --volume-throughput VOLUMETHOUGHPUT Volume throughput for gp3 (default: 125) + --not-encrypted Flag to make the volume un-encyrpted. Default is to create + an encrypted volume + --min-ebs-volume-size SIZE_GB Mimimum size in GB of new volumes created by the instance. (Default: 150) @@ -101,6 +104,7 @@ MAX_EBS_VOLUME_SIZE=1500 MAX_LOGICAL_VOLUME_SIZE=8000 MAX_ATTACHED_VOLUMES=16 INITIAL_UTILIZATION_THRESHOLD=50 +ENCRYPTED=1 IMDSV2="" DEVICE="" @@ -130,6 +134,10 @@ while (( "$#" )); do VOLUMETHOUGHPUT=$2 shift 2 ;; + --not-encrypted) + unset ENCRYPTED + shift + ;; --min-ebs-volume-size) MIN_EBS_VOLUME_SIZE=$2 shift 2 @@ -221,6 +229,7 @@ cat ${BASEDIR}/config/ebs-autoscale.json | \ sed -e "s#%%VOLUMETYPE%%#${VOLUMETYPE}#" | \ sed -e "s#%%VOLUMEIOPS%%#${VOLUMEIOPS}#" | \ sed -e "s#%%VOLUMETHOUGHPUT%%#${VOLUMETHOUGHPUT}#" | \ + sed -e "s#%%ENCRYPTED%%#${ENCRYPTED}#" | \ sed -e "s#%%FILESYSTEM%%#${FILE_SYSTEM}#" | \ sed -e "s#%%MINEBSVOLUMESIZE%%#${MIN_EBS_VOLUME_SIZE}#" | \ sed -e "s#%%MAXEBSVOLUMESIZE%%#${MAX_EBS_VOLUME_SIZE}#" | \ diff --git a/shared/utils.sh b/shared/utils.sh index a86bac2..ce5f4c2 100644 --- a/shared/utils.sh +++ b/shared/utils.sh @@ -49,9 +49,12 @@ function detect_init_system() { # detects the init system in use # based on the following: # https://unix.stackexchange.com/a/164092 - if [[ `/sbin/init --version` =~ upstart ]]; then echo upstart; - elif [[ `systemctl` =~ -\.mount ]]; then echo systemd; - elif [[ -f /etc/init.d/cron && ! -h /etc/init.d/cron ]]; then echo sysv-init; + if [[ $(/sbin/init --version) =~ upstart ]]; then + echo upstart + elif [[ $(systemctl) =~ -\.mount ]]; then + echo systemd + elif [[ -f /etc/init.d/cron && ! -L /etc/init.d/cron ]]; then + echo sysv-init else echo unknown; fi } @@ -62,7 +65,20 @@ function get_config_value() { } function logthis() { - echo "[`date`] $1" >> $(get_config_value .logging.log_file) + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S.%6N%:z") + if [ $# -eq 2 ]; then + level=$2 + else + level="info" + fi + + if command -v logger &>/dev/null; then + message="level=${level} msg=$1" + echo "$message" | tee -a $(get_config_value .logging.log_file) | logger -t ebs-autoscale + else + message="${timestamp} hostname=$(hostname -s) service=ebs-autoscale level=${level} msg=$1" + echo "$message" >>$(get_config_value .logging.log_file) + fi } function starting() {