Merge pull request #1745 from CLHatch/variable-naming

Add "APPNAME" headings in `.env`, and move the "migrate" logic into `appvars_create.sh`

.scripts/appvars_create.sh

@@ -7,24 +7,74 @@ appvars_create() {
     APPNAME=${APPNAME^^}
     local FILENAME=${APPNAME,,}
     local APPTEMPLATES="${SCRIPTPATH}/compose/.apps/${FILENAME}"
+    local APPLABELFILE="${APPTEMPLATES}/${FILENAME}.labels.yml"
+
+    local -A APP_VAR_VALUE
+    local -A APP_VAR_MIGRATE
+
+    # Build variable values lookup array, APP_VAR_VALUES["variable"]="default value"
+    {
+        # Read all lines with labels into temporary APP_LABEL_LINES array
+        local -a APP_LABEL_LINES
+        readarray -t APP_LABEL_LINES < <(grep --color=never -P "\scom\.dockstarter\.appvars\.\K[\w]+" "${APPLABELFILE}" || true)
+        if [[ -z ${APP_LABEL_LINES[*]} ]]; then
+            error "Unable to find labels for ${APPNAME}"
+            return
+        fi
+
+        for line in "${APP_LABEL_LINES[@]}"; do
+            local SET_VAR
+            local SET_VAL
+            SET_VAR=$(echo "$line" | grep --color=never -Po "\scom\.dockstarter\.appvars\.\K[\w]+")
+            SET_VAL=$(echo "$line" | grep --color=never -Po "\scom\.dockstarter\.appvars\.${SET_VAR}: \K.*" | sed -E 's/^([^"].*[^"])$/"\1"/' | xargs || true)
+            if [[ -n ${SET_VAR} ]]; then
+                APP_VAR_VALUE["${SET_VAR^^}"]=${SET_VAL}
+            fi
+        done
+    }
+
+    # Build migrate variable lookup array, APP_MIGRATE_VAR["variable"]="migrate from variable"
+    for SET_VAR in "${!APP_VAR_VALUE[@]}"; do
+        local APPNAME=${SET_VAR%%_*}
+        local REST_VAR=${SET_VAR#"${APPNAME}_"}
+        local VAR_TYPE=${REST_VAR%%_*}
+        case "${VAR_TYPE}" in
+            ENVIRONMENT | VOLUME)
+                REST_VAR=${REST_VAR#"${VAR_TYPE}"}
+                local MIGRATE_VAR="${APPNAME}${REST_VAR}"
+                # shellcheck disable=SC2199
+                if [[ " ${!APP_VAR_VALUE[@]} " != *" ${MIGRATE_VAR} "* ]]; then
+                    # Potential "migrate from" variable isn't an existing app variable, add it to the migrate list
+                    APP_VAR_MIGRATE["${SET_VAR}"]=${MIGRATE_VAR}
+                fi
+                ;;
+        esac
+    done
+
+    # Actual processing starts here
     info "Creating environment variables for ${APPNAME}."
-    while IFS= read -r line; do
-        local VAR_LABEL=${line}
-        local SET_VAR=${VAR_LABEL^^}
-        if grep -q -P "^${SET_VAR}=" "${COMPOSE_ENV}"; then
+    for SET_VAR in "${!APP_VAR_VALUE[@]}"; do
+        if grep -q -P "^\s*${SET_VAR}\s*=" "${COMPOSE_ENV}"; then
+            # Variable already exists
             continue
         fi
 
-        local DEFAULT_VAL
-        DEFAULT_VAL=$(grep --color=never -Po "\scom\.dockstarter\.appvars\.${VAR_LABEL}: \K.*" "${APPTEMPLATES}/${FILENAME}.labels.yml" | sed -E 's/^([^"].*[^"])$/"\1"/' | xargs || true)
-        echo "${SET_VAR}=" >> "${COMPOSE_ENV}"
-        run_script 'env_set' "${SET_VAR}" "${DEFAULT_VAL}"
-    done < <(grep --color=never -Po "\scom\.dockstarter\.appvars\.\K[\w]+" "${APPTEMPLATES}/${FILENAME}.labels.yml" || error "Unable to find labels for ${APPNAME}")
+        local MIGRATE_VAR=${APP_VAR_MIGRATE["${SET_VAR}"]-}
+        if [[ -n ${MIGRATE_VAR} ]] && grep -q -P "^\s*${MIGRATE_VAR}\s*=" "${COMPOSE_ENV}"; then
+            # Migrate old variable
+            run_script 'env_rename' "${MIGRATE_VAR}" "${SET_VAR}"
+        else
+            # Add new variable
+            local DEFAULT_VAL=${APP_VAR_VALUE["${SET_VAR}"]}
+            notice "Adding ${SET_VAR}='${DEFAULT_VAL}' in ${COMPOSE_ENV} file."
+            run_script 'env_set' "${SET_VAR}" "${DEFAULT_VAL}"
+        fi
+    done
     run_script 'env_set' "${APPNAME}_ENABLED" true
 }
 
 test_appvars_create() {
-    run_script 'env_update'
     run_script 'appvars_create' WATCHTOWER
+    run_script 'env_update'
     cat "${COMPOSE_ENV}"
 }

.scripts/appvars_create_all.sh

@@ -16,7 +16,7 @@ appvars_create_all() {
 }
 
 test_appvars_create_all() {
-    run_script 'env_update'
     run_script 'appvars_create_all'
+    run_script 'env_update'
     cat "${COMPOSE_ENV}"
 }

.scripts/appvars_purge.sh

@@ -25,7 +25,7 @@ appvars_purge() {
 }
 
 test_appvars_purge() {
-    run_script 'env_update'
     run_script 'appvars_purge' WATCHTOWER
+    run_script 'env_update'
     cat "${COMPOSE_ENV}"
 }

.scripts/appvars_purge_all.sh

@@ -17,7 +17,7 @@ appvars_purge_all() {
 }
 
 test_appvars_purge_all() {
-    run_script 'env_update'
     run_script 'appvars_purge_all'
+    run_script 'env_update'
     cat "${COMPOSE_ENV}"
 }

.scripts/appvars_rename.sh

@@ -17,7 +17,7 @@ appvars_rename() {
         DOCKER_VOLUME_CONFIG=$(run_script 'env_get' DOCKER_VOLUME_CONFIG)
         mv "${DOCKER_VOLUME_CONFIG}/${FROMAPP,,}" "${DOCKER_VOLUME_CONFIG}/${TOAPP,,}" || warn "Failed to move folder.\nFailing command: ${F[C]}mv \"${DOCKER_VOLUME_CONFIG}/${FROMAPP,,}\" \"${DOCKER_VOLUME_CONFIG}/${TOAPP,,}\""
         notice "Migrating vars."
-        sed -i "s/^${FROMAPP^^}_/${TOAPP^^}_/" "${COMPOSE_ENV}" || fatal "Failed to migrate vars from ${FROMAPP^^}_ to ${TOAPP^^}_\nFailing command: ${F[C]}sed -i \"s/^${FROMAPP^^}_/${TOAPP^^}_/\" \"${COMPOSE_ENV}\""
+        sed -i "s/^\s*${FROMAPP^^}_/${TOAPP^^}_/" "${COMPOSE_ENV}" || fatal "Failed to migrate vars from ${FROMAPP^^}_ to ${TOAPP^^}_\nFailing command: ${F[C]}sed -i \"s/^\\s*${FROMAPP^^}_/${TOAPP^^}_/\" \"${COMPOSE_ENV}\""
         run_script 'appvars_create' "${TOAPP^^}"
         notice "Completed migrating from ${FROMAPP^^} to ${TOAPP^^}. Run ${F[C]}ds -c${NC} to create the new container."
     fi

.scripts/env_get.sh

@@ -5,7 +5,7 @@ IFS=$'\n\t'
 env_get() {
     local GET_VAR=${1-}
     local VAR_FILE=${2:-$COMPOSE_ENV}
-    grep --color=never -Po "^${GET_VAR}=\K.*" "${VAR_FILE}" | xargs || true
+    grep --color=never -Po "^\s*${GET_VAR}\s*=\K.*" "${VAR_FILE}" | tail -1 | xargs || true
 }
 
 test_env_get() {

.scripts/env_rename.sh

@@ -5,9 +5,9 @@ IFS=$'\n\t'
 env_rename() {
     local FROMVAR=${1-}
     local TOVAR=${2-}
-    if grep -q -P "^${FROMVAR^^}=" "${COMPOSE_ENV}"; then
-        notice "Renaming ${FROMVAR^^} to ${TOVAR^^} in ${COMPOSE_ENV} file."
-        sed -i "s/^${FROMVAR^^}=/${TOVAR^^}=/" "${COMPOSE_ENV}" || fatal "Failed to rename var from ${FROMVAR^^} to ${TOVAR^^}\nFailing command: ${F[C]}sed -i \"s/^${FROMVAR^^}=/${TOVAR^^}=/\" \"${COMPOSE_ENV}\""
+    if grep -q -P "^\s*${FROMVAR}\s*=" "${COMPOSE_ENV}"; then
+        notice "Renaming ${FROMVAR} to ${TOVAR} in ${COMPOSE_ENV} file."
+        sed -i "s/^\s*${FROMVAR}\s*=/${TOVAR}=/" "${COMPOSE_ENV}" || fatal "Failed to rename var from ${FROMVAR} to ${TOVAR}\nFailing command: ${F[C]}sed -i \"s/^\\s*${FROMVAR}\\s*=/${TOVAR}=/\" \"${COMPOSE_ENV}\""
     fi
 }
 

.scripts/env_sanitize.sh

@@ -21,9 +21,6 @@ env_sanitize() {
     run_script 'appvars_rename' MINECRAFT_BEDROCK_SERVER MINECRAFTBEDROCKSERVER
     run_script 'appvars_rename' MINECRAFT_SERVER MINECRAFTSERVER
 
-    # Migrate from old app vars
-    run_script 'appvars_migrate_all'
-
     # Replace ~ with /home/username
     if grep -q -P '^\w+_VOLUME_\w+=~/' "${COMPOSE_ENV}"; then
         info "Replacing ~ with ${DETECTED_HOMEDIR} in ${COMPOSE_ENV} file."

.scripts/env_set.sh

@@ -8,14 +8,9 @@ env_set() {
     # https://unix.stackexchange.com/questions/422165/escape-double-quotes-in-variable/422170#422170
     NEW_VAL=$(printf "%s\n" "${2-}" | sed -e "s/'/'\"'\"'/g" -e "1s/^/'/" -e "\$s/\$/'/")
     local VAR_FILE=${3:-$COMPOSE_ENV}
-    local VAR_VAL
-    VAR_VAL=$(grep --color=never -P "^${SET_VAR}=" "${VAR_FILE}") || fatal "Failed to find ${SET_VAR} in ${VAR_FILE}\nFailing command: ${F[C]}grep --color=never -P \"^${SET_VAR}=\" \"${VAR_FILE}\""
-    # https://stackoverflow.com/questions/29613304/is-it-possible-to-escape-regex-metacharacters-reliably-with-sed/29613573#29613573
-    local SED_FIND
-    SED_FIND=$(sed 's/[^^]/[&]/g; s/\^/\\^/g' <<< "${VAR_VAL}")
-    local SED_REPLACE
-    SED_REPLACE=$(sed 's/[&/\]/\\&/g' <<< "${SET_VAR}=${NEW_VAL}")
-    sed -i "s/^${SED_FIND}$/${SED_REPLACE}/" "${VAR_FILE}" || fatal "Failed to set ${SED_REPLACE}\nFailing command: ${F[C]}sed -i \"s/^${SED_FIND}$/${SED_REPLACE}/\" \"${VAR_FILE}\""
+
+    sed -i "/^\s*${SET_VAR}\s*=/d" "${VAR_FILE}" || true
+    echo "${SET_VAR}=${NEW_VAL}" >> "${VAR_FILE}" || fatal "Failed to set ${SET_VAR}=${NEW_VAL}\nFailing command: ${F[C]} \"echo ${SET_VAR}=${NEW_VAL}\" >> \"${VAR_FILE}\""
 }
 
 test_env_set() {

.scripts/env_update.sh

@@ -5,25 +5,144 @@ IFS=$'\n\t'
 env_update() {
     run_script 'env_backup'
     run_script 'override_backup'
+
     info "Replacing current .env file with latest template."
-    local MKTEMP_ENV_CURRENT
-    MKTEMP_ENV_CURRENT=$(mktemp) || fatal "Failed to create temporary current .env file.\nFailing command: ${F[C]}mktemp"
-    sort "${COMPOSE_ENV}" > "${MKTEMP_ENV_CURRENT}" || fatal "Failed to sort to new file.\nFailing command: ${F[C]}sort \"${COMPOSE_ENV}\" > \"${MKTEMP_ENV_CURRENT}\""
+
+    # Current .env file, variables only (remove whitespace before and after variable)
+    local -a CURRENT_ENV_LINES
+    readarray -t CURRENT_ENV_LINES < <(sed -n "s/^\s*\([A-Za-z0-9_]*\)\s*=/\1=/p" "${COMPOSE_ENV}")
+
+    # New .env file we are creating
+    local -a UPDATED_ENV_LINES
+    readarray -t UPDATED_ENV_LINES < "${COMPOSE_ENV}.example"
+
+    # CURRENT_ENV_VAR_LINE["VAR"]="line"
+    local -A CURRENT_ENV_VAR_LINE
+    {
+        local -a UPDATED_ENV_LINES_STRIPPED
+        readarray -t UPDATED_ENV_LINES_STRIPPED < <(printf '%s\n' "${UPDATED_ENV_LINES[@]}" | grep -v '^#' | grep '=')
+        for line in "${UPDATED_ENV_LINES_STRIPPED[@]}" "${CURRENT_ENV_LINES[@]}"; do
+            local VAR=${line%%=*}
+            CURRENT_ENV_VAR_LINE[$VAR]=$line
+        done
+    }
+
+    # UPDATED_ENV_VAR_INDEX["VAR"]=index position of line in UPDATED_ENV_LINE
+    local -A UPDATED_ENV_VAR_INDEX
+    {
+        local -a VAR_LINES
+        # Make an array with the contents "line number:VARIABLE" in each element
+        readarray -t VAR_LINES < <(printf '%s\n' "${UPDATED_ENV_LINES[@]}" | grep -n -o -P '^[A-Za-z0-9_]*(?=[=])')
+        for line in "${VAR_LINES[@]}"; do
+            local index=${line%:*}
+            index=$((index - 1))
+            local VAR=${line#*:}
+            UPDATED_ENV_VAR_INDEX[$VAR]=$index
+        done
+    }
+
+    info "Merging current values into updated .env file."
+
+    local BUILTIN_APPS=()
+    local INSTALLED_APPS=()
+    local APPTEMPLATESFOLDER="${SCRIPTPATH}/compose/.apps"
+
+    # Create array of built in apps
+    readarray -t BUILTIN_APPS < <(find "${APPTEMPLATESFOLDER}" -maxdepth 1 -mindepth 1 -type d -exec basename {} \;)
+
+    # Create array of installed apps
+    {
+        local ENABLED_LINES=()
+        readarray -t ENABLED_LINES < <(printf '%s\n' "${CURRENT_ENV_LINES[@]}" | grep --color=never -P '^[A-Za-z0-9]\w+_ENABLED=')
+        for line in "${ENABLED_LINES[@]}"; do
+            local VAR=${line%%=*}
+            local APPNAME=${VAR%%_*}
+            # shellcheck disable=SC2199
+            if [[ " ${BUILTIN_APPS[@]^^} " == *" ${APPNAME} "* ]]; then
+                INSTALLED_APPS+=("${APPNAME}")
+            fi
+        done
+    }
+
+    # Create sorted array of vars in current .env file.  Sort `_` to the top.
+    local -a CURRENT_ENV_VARS
+    readarray -t CURRENT_ENV_VARS < <(printf '%s\n' "${!CURRENT_ENV_VAR_LINE[@]}" | tr "_" "." | env LC_ALL=C sort | tr "." "_")
+    # Process each variable, adding them to the updated .env array
+    while [[ -n ${CURRENT_ENV_VARS[*]} ]]; do
+        # Loop while there are variables in array
+        local APPNAME
+        local LAST_APPNAME
+
+        # Clear lists before processing an app's variables
+        local APP_LABELS=()
+        local ENV_VARS_BUILTIN=()
+        local ENV_VARS_USER_DEFINED=()
+
+        # Process lines for one app
+        for index in "${!CURRENT_ENV_VARS[@]}"; do
+            VAR=${CURRENT_ENV_VARS[$index]}
+            APPNAME=${VAR%%_*}
+            if [ "${APPNAME}" != "${LAST_APPNAME-}" ]; then
+                # Variable for another app, exit for loop
+                break
+            fi
+            if [[ -n ${UPDATED_ENV_VAR_INDEX["$VAR"]-} ]]; then
+                # Variable already exists, update its value
+                UPDATED_ENV_LINES[${UPDATED_ENV_VAR_INDEX["$VAR"]}]=${CURRENT_ENV_VAR_LINE["$VAR"]}
+            else
+                # Variable does not already exist, add it to a list to process
+                if [[ -z ${APP_LABELS[*]} ]]; then
+                    # App label array is empty, create it
+                    # shellcheck disable=SC2199
+                    if [[ " ${INSTALLED_APPS[@]} " == *" ${APPNAME} "* ]]; then
+                        # Create array of labels for current app being processed
+                        local APPTEMPLATE="${APPTEMPLATESFOLDER}/${APPNAME,,}/${APPNAME,,}.labels.yml"
+                        readarray -t APP_LABELS < <(grep --color=never -Po "\scom\.dockstarter\.appvars\.\K[\w]+" "${APPTEMPLATE}" || true)
+                        APP_LABELS=("${APP_LABELS[@]^^}")
+                    fi
+                fi
+                # shellcheck disable=SC2199
+                if [[ " ${APP_LABELS[@]} " == *" ${VAR} "* ]]; then
+                    # Variable is in label file, add it to the built in list
+                    ENV_VARS_BUILTIN+=("$VAR")
+                else
+                    # Variable is not in label file, add it to the user defined list
+                    ENV_VARS_USER_DEFINED+=("$VAR")
+                fi
+            fi
+            # Remove processed var from array
+            unset 'CURRENT_ENV_VARS[index]'
+        done
+
+        # Add the lines to the env file from the built in list and user defined list for the last app being processed
+        # shellcheck disable=SC2034  # Variable is used indirectly
+        local HEADING_FORMAT_BUILTIN="\n##\n## %s\n##"
+        # shellcheck disable=SC2034  # Variable is used indirectly
+        local HEADING_FORMAT_USER_DEFINED="\n##\n## %s (User Defined)\n##"
+        for section in BUILTIN USER_DEFINED; do
+            local VARS="ENV_VARS_${section}[@]"
+            if [[ -n ${!VARS-} ]]; then
+                # Add all app variables for section to updated .env
+                local HEADING_FORMAT="HEADING_FORMAT_${section}"
+                local HEADING
+                # shellcheck disable=SC2059 # ${!HEADING_FORMAT} contains a printf format string
+                printf -v HEADING "${!HEADING_FORMAT}" "${LAST_APPNAME-}"
+                UPDATED_ENV_LINES+=("${HEADING}")
+                for VAR in "${!VARS}"; do
+                    UPDATED_ENV_LINES+=("${CURRENT_ENV_VAR_LINE[$VAR]}")
+                    #UPDATED_ENV_VAR_INDEX[$VAR]=$((${#UPDATED_ENV_LINES[@]} - 1))
+                done
+            fi
+        done
+
+        # Set last app worked on
+        LAST_APPNAME=${APPNAME}
+    done
+
     local MKTEMP_ENV_UPDATED
     MKTEMP_ENV_UPDATED=$(mktemp) || fatal "Failed to create temporary update .env file.\nFailing command: ${F[C]}mktemp"
-    cp "${COMPOSE_ENV}.example" "${MKTEMP_ENV_UPDATED}" || fatal "Failed to copy file.\nFailing command: ${F[C]}cp \"${COMPOSE_ENV}.example\" \"${MKTEMP_ENV_UPDATED}\""
-    info "Merging current values into updated .env file."
-    while IFS= read -r line; do
-        local VAR_VAL=${line}
-        local SET_VAR=${VAR_VAL%%=*}
-        local SET_VAL
-        SET_VAL=$(run_script 'env_get' "${SET_VAR}" "${MKTEMP_ENV_CURRENT}")
-        if ! grep -q -P "^${SET_VAR}=" "${MKTEMP_ENV_UPDATED}"; then
-            echo "${VAR_VAL}" >> "${MKTEMP_ENV_UPDATED}" || error "${VAR_VAL} could not be written to ${MKTEMP_ENV_UPDATED}"
-        fi
-        run_script 'env_set' "${SET_VAR}" "${SET_VAL}" "${MKTEMP_ENV_UPDATED}"
-    done < <(grep -v -P '^#' "${MKTEMP_ENV_CURRENT}" | grep '=')
-    rm -f "${MKTEMP_ENV_CURRENT}" || warn "Failed to remove temporary .env update file.\nFailing command: ${F[C]}rm -f \"${MKTEMP_ENV_CURRENT}\""
+    printf '%s\n' "${UPDATED_ENV_LINES[@]}" > "${MKTEMP_ENV_UPDATED}" || fatal "Failed to write temporary .env update file."
+
     cp -f "${MKTEMP_ENV_UPDATED}" "${COMPOSE_ENV}" || fatal "Failed to copy file.\nFailing command: ${F[C]}cp -f \"${MKTEMP_ENV_UPDATED}\" \"${COMPOSE_ENV}\""
     rm -f "${MKTEMP_ENV_UPDATED}" || warn "Failed to remove temporary .env update file.\nFailing command: ${F[C]}rm -f \"${MKTEMP_ENV_UPDATED}\""
     run_script 'set_permissions' "${COMPOSE_ENV}"

.scripts/yml_merge.sh

@@ -3,8 +3,8 @@ set -Eeuo pipefail
 IFS=$'\n\t'
 
 yml_merge() {
-    run_script 'env_update'
     run_script 'appvars_create_all'
+    run_script 'env_update'
     info "Compiling enabled templates to merge docker-compose.yml file."
     local COMPOSE_FILE
     COMPOSE_FILE="${SCRIPTPATH}/compose/.reqs/r1.yml"