Merge pull request #212 from amiaopensource/formatlto-updates

Formatlto updates

collectionchecksum

@@ -6,45 +6,99 @@
 # presumes a highly specific directory structure based on CUNY TV's Archival
 # Information Package directory structure.
 
-SCRIPTDIR=$(dirname "${0}")
+_get_iso8601(){
+    date +%FT%T
+}
 
-. "${SCRIPTDIR}/mmfunctions" || { echo "Missing '${SCRIPTDIR}/mmfunctions'. Exiting." ; exit 1 ; }
+_report(){
+    local RED="$(tput setaf 1)"   # Red      - For Warnings
+    local GREEN="$(tput setaf 2)" # Green    - For Declarations
+    local BLUE="$(tput setaf 4)"  # Blue     - For Questions
+    local NC="$(tput sgr0)"       # No Color
+    local COLOR=""
+    local STARTMESSAGE=""
+    local ECHOOPT=""
+    OPTIND=1
+    while getopts ":qdwstn" OPT; do
+        case "${OPT}" in
+            q) COLOR="${BLUE}" ;;                         # question mode, use color blue
+            d) COLOR="${GREEN}" ;;                        # declaration mode, use color green
+            w) COLOR="${RED}" ;;                          # warning mode, use color red
+            s) STARTMESSAGE+=([$(basename "${0}")] ) ;;   # prepend scriptname to the message
+            t) STARTMESSAGE+=($(_get_iso8601) '- ' ) ;;   # prepend timestamp to the message
+            n) ECHOOPT="-n" ;;                            # to avoid line breaks after echo
+        esac
+    done
+    shift $(( OPTIND - 1 ))
+    MESSAGE="${1}"
+    echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[*]}${MESSAGE}${NC}"
+}
 
 _maketemp(){
-    mktemp -q "/tmp/$(basename "${0}").XXXXXX"
+    mktemp -t "$(basename "${0}")"
     if [ "${?}" -ne 0 ] ; then
         _report -w "${0}: Can't create temp file. Exiting."
-        _writeerrorlog "_maketemp" "was unable to create the temp file, so the script had to exit."
+        _report -w "_maketemp was unable to create the temp file, so the script had to exit."
         exit 1
     fi
 }
 
 while [ "${*}" != "" ] ; do
     INPUT="${1}"
-    _report_to_db
     TAPECHECKSUMFILE="${INPUT}/tapechecksum.md5"
+    TAPEWRITELOG="${INPUT}/tapewritelog.txt"
+    TEMP_CHECKSUM_COMPILATION="$(_maketemp)"
     if [ -f "${TAPECHECKSUMFILE}" ] ; then
         _report -wt "A collection checksum already exists at ${TAPECHECKSUMFILE}"
         exit 1
     fi
-    TEMPCHECKSUMFILE=$(_maketemp)
-    cd "${INPUT}"
+    _report -d -n "Concatenating existing checksum.md5 files ... "
+    find "${INPUT}" -type f -name checksum.md5 | while read CHECKSUM_FILE ; do
+        UPF="$(basename "$(dirname "${CHECKSUM_FILE}")")"
+        if [[ "${UPF}" == "metadata" ]] ; then
+            PATH_CLEANING="$(basename "$(dirname "$(dirname "${CHECKSUM_FILE}")")")/objects/"
+        else
+            PATH_CLEANING="$(echo "${CHECKSUM_FILE}" | sed "s|${INPUT}/||g;s|checksum.md5||g")"
+        fi
+        sed "s|  ./|  ./${PATH_CLEANING}|g" "${CHECKSUM_FILE}"
+    done > "${TEMP_CHECKSUM_COMPILATION}"
+    _report -d "$(wc -l "${TEMP_CHECKSUM_COMPILATION}" | awk '{print $1}') checksum.md5 were gathered for ${INPUT}."
 
-    #loop through the directories, find the checksum files, cat them together, and also sed to add the media ids
-    while read CHECKSUMFILES ; do
-        PACKAGE="$(basename "$(dirname "$(dirname "${CHECKSUMFILES}")")")"
-        _report -d "Concatenating ${CHECKSUMFILES}."
-        cat "${CHECKSUMFILES}" | sed "s|  ./|  ./${PACKAGE}/objects/|g" >> "${TEMPCHECKSUMFILE}"
-    done < <(find "${INPUT}" -maxdepth 3 -mindepth 3 -type f -iname "checksum.md5")
+    if [[ -d "${INPUT}" ]] ; then
+        cd "${INPUT}"
+        TEMP_FILE_LIST="$(_maketemp)"
+        TEMP_LIST2MD5="$(_maketemp)"
+        find . -type f ! -name ".DS_Store" > "${TEMP_FILE_LIST}"
+        LIST_COMPARE="$(diff <(cut -c 35- "${TEMP_CHECKSUM_COMPILATION}" | sort) <(sort "${TEMP_FILE_LIST}"))"
+        MISSING_FILES="$(echo "${LIST_COMPARE}" | grep "^<")"
+        echo "${LIST_COMPARE}" | grep "^>" | cut -c 3- > "${TEMP_LIST2MD5}"
+        if [[ -n "${MISSING_FILES}" ]] ; then
+            _report -w "Warning there are files that are in the checksum.md5 files but not present"
+            echo "${MISSING_FILES}"
+            exit 1
+        fi
+        TEMP_NEWMD5S="$(_maketemp)"
+        if [[ -s "${TEMP_LIST2MD5}" ]] ; then
+            _report -d "For the tape checksum, these $(wc -l "${TEMP_LIST2MD5}" | awk '{print $1}') additional files must be checksummed"
+            cat "${TEMP_LIST2MD5}"
+            md5deep -rel -f "${TEMP_LIST2MD5}" > "${TEMP_NEWMD5S}"
+            _report -d "Checksummed $(wc -l "${TEMP_CHECKSUM_COMPILATION}" | awk '{print $1}') additional files."
+        fi
+        cat "${TEMP_NEWMD5S}" "${TEMP_CHECKSUM_COMPILATION}" | sort -k2 > "${TAPECHECKSUMFILE}"
+        _report -d "${TAPECHECKSUMFILE} is ready with $(wc -l "${TAPECHECKSUMFILE}" | awk '{print $1}') checksums."
+    else
+        _report -d "${INPUT} is not a directory. Exiting."
+        exit 1
+    fi
 
-    #create md5 checksums for all of the files in the metadata directory
-    _report -d "Creating checksums for all metadata directories."
-    md5deep -rel ./*/metadata >> "${TEMPCHECKSUMFILE}"
+    _report -q -n "Name of operator: "
+    read OP
+    _report -q -n "Note about tape write: "
+    read NOTE
 
-    _report -d "Sorting and saving ${TAPECHECKSUMFILE}."
-    sort -k2 "${TEMPCHECKSUMFILE}" | grep -v .DS_Store >> "${TAPECHECKSUMFILE}"
+    echo "operator: ${OP}" > "${TAPEWRITELOG}"
+    echo "datetime: $(_get_iso8601)" >> "${TAPEWRITELOG}"
+    echo "note: ${NOTE}" >> "${TAPEWRITELOG}"
 
     shift
-    _eventoutcome_update
-    _report_fixity_db
 done

formatlto

@@ -4,21 +4,62 @@
 # Formats an LTO tape with LTFS.
 
 SCRIPTDIR=$(dirname "${0}")
-DEPENDENCIES=(mkltfs mmfunctions)
+DEPENDENCIES=(mkltfs)
 TAPE_SERIAL_REGEX="^[A-Z0-9]{6}$"
 BLUE="$(tput setaf 4)"  # Blue     - For Questions
 NC="$(tput sgr0)"       # No Color
 
-. "${SCRIPTDIR}/mmfunctions" || { echo "Missing '${SCRIPTDIR}/mmfunctions'. Exiting." ; exit 1 ; }
+_check_dependencies(){
+    DEPS_OK=YES
+    while [ "${*}" != "" ] ; do
+        DEPENDENCY="${1}"
+        if [ ! "$(which "${DEPENDENCY}")" ] ; then
+            _report -wt "This script requires ${DEPENDENCY} to run but it is not installed"
+            _report -wt "If you are running ubuntu or debian you might be able to install ${DEPENDENCY} with the following command"
+            _report -wt "sudo apt-get install ${DEPENDENCY}"
+            _report -wt "If you are running mac you might be able to install ${DEPENDENCY} with the following command"
+            _report -wt "brew install ${DEPENDENCY}"
+            DEPS_OK=NO
+        fi
+        shift
+    done
+    if [[ "${DEPS_OK}" = "NO" ]]; then
+        _report -wt "Unmet dependencies"
+        _report -wt "Aborting!"
+        exit 1
+    else
+        return 0
+    fi
+}
 
-unset VERSION
-if [[ $(dirname $(command -v "${0}")) = "/usr/local/bin" || $(dirname $(command -v "${0}")) = "${HOME}/.linuxbrew/bin" ]] ; then
-    VERSION=$(TMP=$(brew info ltopers | grep ".*\*$" | grep -Eo "/ltopers/.* \(") ; echo "${TMP:9:(${#TMP}-11)}")
-fi
+_report(){
+    local RED="$(tput setaf 1)"   # Red      - For Warnings
+    local GREEN="$(tput setaf 2)" # Green    - For Declarations
+    local BLUE="$(tput setaf 4)"  # Blue     - For Questions
+    local NC="$(tput sgr0)"       # No Color
+    local COLOR=""
+    local STARTMESSAGE=""
+    local ECHOOPT=""
+    local LOG_MESSAGE=""
+    OPTIND=1
+    while getopts ":qdwstn" OPT; do
+        case "${OPT}" in
+            q) COLOR="${BLUE}" ;;                         # question mode, use color blue
+            d) COLOR="${GREEN}" ;;                        # declaration mode, use color green
+            w) COLOR="${RED}" ;;                          # warning mode, use color red
+            s) STARTMESSAGE+=([$(basename "${0}")] ) ;;   # prepend scriptname to the message
+            t) STARTMESSAGE+=($(_get_iso8601) '- ' ) ;;   # prepend timestamp to the message
+            n) ECHOOPT="-n" ;;                            # to avoid line breaks after echo
+        esac
+    done
+    shift $(( OPTIND - 1 ))
+    MESSAGE="${1}"
+    echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[*]}${MESSAGE}${NC}"
+}
 
 _usage(){
     cat <<EOF
-$(basename "${0}") ${VERSION}
+$(basename "${0}")
 This script formats an LTO tape with LTFS.
 Dependencies: ${DEPENDENCIES[@]}
 
@@ -41,6 +82,8 @@ while getopts ":fcxh" opt ; do
     esac
 done
 
+_check_dependencies "${DEPENDENCIES[@]}"
+
 if [ "${FORCE}" = "1" ] ; then
     MIDDLE_OPTIONS+=(-f)
     _report -d "Will force formatting."
@@ -52,32 +95,48 @@ else
     _report -d "Will use compression."
 fi
 
-if [ ! "${LTO_ARRAY}" ] ; then
-    if [[ "$(uname -s)" = "Darwin" ]] ; then
-        # try to figure out how many LTO decks are attached
-        LTO_ARRAY=($(system_profiler SPSASDataType | grep "SCSI Target Identifier" | cut -d : -f2 | sort | xargs))
+LTO_LIST="$(ltfs -o device_list 2>&1 | grep "Device Name = [0-9]*")"
+# try to figure out how many lto decks are attached
+LTO_COUNT=$(echo -n "${LTO_LIST}" | grep -c '^')
+if [[ "${LTO_COUNT}" -eq 0 ]] ; then
+    if [[ "$(ltfs -h 2>&1 | grep -c device_list)" -eq 0 ]] ; then
+        _report -w "The installed version of ltfs at $(which ltfs) doesn't support the '-o device_list' option. Please upgrade ltfs to 2.4.4.0 or newer."
+        exit 1
     else
-        #### TO DO: itemize when there are multiple LTO drives in Linux and Windows
-        LTO_ARRAY=(0)
+        _report -w "No available LTO decks were found or they're all busy."
+        df -Ph | grep "^Filesystem\|^ltfs" | sort
+        exit 1
     fi
 fi
 
-if [[ "${#LTO_ARRAY[@]}" -gt 1 ]] ; then
-    PS3="${BLUE}Which LTO deck?${NC} "
-    eval set "${LTO_ARRAY[@]}"
-    select DECK in "$@" ; do
-       break
-    done
-    if [ ! "${DECK}" ] ; then
-        _report -w "Error: You selected an invalid deck."
-        exit 1
+_report -d "Checking for ready decks amongst..."
+printf "%-12s %20s %20s \n" "Device ID" "Product ID" "Serial Number"
+echo "${LTO_LIST}" | while read LTO_RECORD ; do
+    DEVICE_ID="$(echo "${LTO_RECORD}" | grep -o "Device Name = [^ ,]*" | cut -d = -f2 | sed 's/ //g')"
+    PRODUCT_ID="$(echo "${LTO_RECORD}" | grep -o "Product ID = [^ ,]*" | cut -d = -f2 | sed 's/ //g')"
+    SERIAL="$(echo "${LTO_RECORD}" | grep -o "Serial Number = [^ ,]*" | cut -d = -f2 | sed 's/ //g')"
+    printf "%-12s %20s %20s \n" "${DEVICE_ID}" "${PRODUCT_ID}" "${SERIAL}"
+done
+echo
+
+if [[ "${LTO_COUNT}" -eq 1 ]] ; then
+    DEVICE_ID="$(echo "${LTO_LIST}" | grep -o "Device Name = [^ ,]*" | cut -d = -f2 | sed 's/ //g')"
+    PRODUCT_ID="$(echo "${LTO_LIST}" | grep -o "Product ID = [^ ,]*" | cut -d = -f2 | sed 's/ //g')"
+    SERIAL="$(echo "${LTO_LIST}" | grep -o "Serial Number = [^ ,]*" | cut -d = -f2 | sed 's/ //g')"
+    if [[ "$(uname -s)" = "Linux" ]] ; then
+        DECK="/dev/sg3"
+        _report -d "Will use ${DECK} to format the tape."
+    else
+        DECK="${DEVICE_ID}"
+        _report -d "Will use Deck ${DECK} (${SERIAL}) to format the tape."
     fi
 else
     #### TO DO: select the correct path to the drive on Linux and Windows
     if [[ "$(uname -s)" = "Linux" ]] ; then
         DECK='/dev/sg3'
     else
-        DECK=0
+        _report -q -n "Enter the Device ID for the deck to use: "
+        read DECK
     fi
 fi
 

mountlto

@@ -76,10 +76,15 @@ _report(){
 }
 
 _maketemp(){
-    mktemp -q -t "$(basename "${0}")"
+    if [[ -n "${1}" ]] ; then
+        PREFIX="$(basename "${0}").${1}"
+    else
+        PREFIX="$(basename "${0}")"
+    fi
+    mktemp -q -t "${PREFIX}"
     if [ "${?}" -ne 0 ]; then
         echo "${0}: Can't create temp file, exiting..."
-        _report -w "_maketemp" "was unable to create the temp file, so the script had to exit."
+        _report -w "_maketemp was unable to create the temp file, so the script had to exit."
         exit 1
     fi
 }
@@ -115,15 +120,16 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do
     if [ "$(command -v ltfs_ldun)" ] ; then
         ltfs_ldun load "${DEVICE_ID}" 1>/dev/null
     fi
-    LOADTMP=$(_maketemp)
+    LOADTMP=$(_maketemp "${SERIAL}")
     unset TAPE_SERIAL
     COUNTER=0
-    echo -n "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})"
-    while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 5 ]]; do
+    _report -dn "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})"
+    while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 2 ]]; do
         ((COUNTER++))
         ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}"
+        LTFS_ERR="${?}"
         # find barcode by either "Volser(Barcode)" (Quantum style) or "Tape attribute: Barcode" (IBM style)
-        TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1)
+        TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*\|Volume mounted successfully*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1)
         echo -n "."
     done
     echo
@@ -135,7 +141,7 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do
             unset TAPE_SERIAL
         fi
     else
-        echo "Deck ${DEVICE_ID} (${SERIAL}) found with tape ${TAPE_SERIAL}."
+        _report -d "Deck ${DEVICE_ID} (${SERIAL}) found with tape ${TAPE_SERIAL}."
         echo -n -e "\033]0;Deck:${SERIAL} LTO:${TAPE_SERIAL}\007"
         if [[ $(uname -s) = "Darwin" ]] ; then
             MOUNT_DIR="/Volumes/${TAPE_SERIAL}"