From 02985ec31af3dd3463a9e72357d0a03603a6f39c Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Fri, 9 Jun 2023 12:04:14 -0400 Subject: [PATCH 01/21] remove slow process to note version --- formatlto | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/formatlto b/formatlto index 180fcd3..da5e572 100755 --- a/formatlto +++ b/formatlto @@ -11,14 +11,9 @@ NC="$(tput sgr0)" # No Color . "${SCRIPTDIR}/mmfunctions" || { echo "Missing '${SCRIPTDIR}/mmfunctions'. Exiting." ; exit 1 ; } -unset VERSION -if [[ $(dirname $(command -v "${0}")) = "/usr/local/bin" || $(dirname $(command -v "${0}")) = "${HOME}/.linuxbrew/bin" ]] ; then - VERSION=$(TMP=$(brew info ltopers | grep ".*\*$" | grep -Eo "/ltopers/.* \(") ; echo "${TMP:9:(${#TMP}-11)}") -fi - _usage(){ cat < Date: Fri, 9 Jun 2023 12:08:05 -0400 Subject: [PATCH 02/21] remove dependency on mmfunctions --- formatlto | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/formatlto b/formatlto index da5e572..ec8e862 100755 --- a/formatlto +++ b/formatlto @@ -4,12 +4,58 @@ # Formats an LTO tape with LTFS. SCRIPTDIR=$(dirname "${0}") -DEPENDENCIES=(mkltfs mmfunctions) +DEPENDENCIES=(mkltfs) TAPE_SERIAL_REGEX="^[A-Z0-9]{6}$" BLUE="$(tput setaf 4)" # Blue - For Questions NC="$(tput sgr0)" # No Color -. "${SCRIPTDIR}/mmfunctions" || { echo "Missing '${SCRIPTDIR}/mmfunctions'. Exiting." ; exit 1 ; } +_check_dependencies(){ + DEPS_OK=YES + while [ "${*}" != "" ] ; do + DEPENDENCY="${1}" + if [ ! "$(which "${DEPENDENCY}")" ] ; then + _report -wt "This script requires ${DEPENDENCY} to run but it is not installed" + _report -wt "If you are running ubuntu or debian you might be able to install ${DEPENDENCY} with the following command" + _report -wt "sudo apt-get install ${DEPENDENCY}" + _report -wt "If you are running mac you might be able to install ${DEPENDENCY} with the following command" + _report -wt "brew install ${DEPENDENCY}" + DEPS_OK=NO + fi + shift + done + if [[ "${DEPS_OK}" = "NO" ]]; then + _report -wt "Unmet dependencies" + _report -wt "Aborting!" + exit 1 + else + return 0 + fi +} + +_report(){ + local RED="$(tput setaf 1)" # Red - For Warnings + local GREEN="$(tput setaf 2)" # Green - For Declarations + local BLUE="$(tput setaf 4)" # Blue - For Questions + local NC="$(tput sgr0)" # No Color + local COLOR="" + local STARTMESSAGE="" + local ECHOOPT="" + local LOG_MESSAGE="" + OPTIND=1 + while getopts ":qdwstn" OPT; do + case "${OPT}" in + q) COLOR="${BLUE}" ;; # question mode, use color blue + d) COLOR="${GREEN}" ;; # declaration mode, use color green + w) COLOR="${RED}" ;; # warning mode, use color red + s) STARTMESSAGE+=([$(basename "${0}")] ) ;; # prepend scriptname to the message + t) STARTMESSAGE+=($(_get_iso8601) '- ' ) ;; # prepend timestamp to the message + n) ECHOOPT="-n" ;; # to avoid line breaks after echo + esac + done + shift $(( OPTIND - 1 )) + MESSAGE="${1}" + echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[*]}${MESSAGE}${NC}" +} _usage(){ cat < Date: Fri, 9 Jun 2023 15:50:15 -0400 Subject: [PATCH 03/21] remove slow version labelling --- writelto | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/writelto b/writelto index 7dbf000..f9b49f8 100755 --- a/writelto +++ b/writelto @@ -14,11 +14,6 @@ TAPE_EJECT="Y" _check_for_lto_index_dir LTO_LOGS="${LTO_INDEX_DIR}" -unset VERSION -if [[ $(dirname $(command -v "${0}")) = "/usr/local/bin" || $(dirname $(command -v "${0}")) = "${HOME}/.linuxbrew/bin" ]] ; then - VERSION=$(TMP=$(brew info ltopers | grep ".*\*$" | grep -Eo "/ltopers/.* \(") ; echo "${TMP:9:(${#TMP}-11)}") -fi - # no .DS_Store onto external storage if [[ "$(uname -s)" = "Darwin" ]] ; then if [[ "$(defaults read com.apple.desktopservices DSDontWriteNetworkStores 2>/dev/null)" != "true" ]] ; then @@ -28,7 +23,7 @@ fi _usage(){ cat < Date: Mon, 12 Jun 2023 13:51:56 -0400 Subject: [PATCH 04/21] prefix temp file with deck serial for easier debugging --- mountlto | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mountlto b/mountlto index 4fc7292..fb76b59 100755 --- a/mountlto +++ b/mountlto @@ -76,7 +76,12 @@ _report(){ } _maketemp(){ - mktemp -q -t "$(basename "${0}")" + if [[ -n "${1}" ]] ; then + PREFIX="$(basename "${0}").${1}" + else + PREFIX="$(basename "${0}")" + fi + mktemp -q -t "${PREFIX}" if [ "${?}" -ne 0 ]; then echo "${0}: Can't create temp file, exiting..." _report -w "_maketemp" "was unable to create the temp file, so the script had to exit." @@ -115,7 +120,7 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do if [ "$(command -v ltfs_ldun)" ] ; then ltfs_ldun load "${DEVICE_ID}" 1>/dev/null fi - LOADTMP=$(_maketemp) + LOADTMP=$(_maketemp "${SERIAL}") unset TAPE_SERIAL COUNTER=0 echo -n "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})" From ef98dd64ccf2ce7d26dc624a9c74b3fd8d60859b Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Mon, 12 Jun 2023 14:17:36 -0400 Subject: [PATCH 05/21] update a few logging messages --- mountlto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mountlto b/mountlto index fb76b59..2b2f199 100755 --- a/mountlto +++ b/mountlto @@ -84,7 +84,7 @@ _maketemp(){ mktemp -q -t "${PREFIX}" if [ "${?}" -ne 0 ]; then echo "${0}: Can't create temp file, exiting..." - _report -w "_maketemp" "was unable to create the temp file, so the script had to exit." + _report -w "_maketemp was unable to create the temp file, so the script had to exit." exit 1 fi } @@ -123,7 +123,7 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do LOADTMP=$(_maketemp "${SERIAL}") unset TAPE_SERIAL COUNTER=0 - echo -n "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})" + _report -d "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})" while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 5 ]]; do ((COUNTER++)) ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}" @@ -140,7 +140,7 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do unset TAPE_SERIAL fi else - echo "Deck ${DEVICE_ID} (${SERIAL}) found with tape ${TAPE_SERIAL}." + _report -d "Deck ${DEVICE_ID} (${SERIAL}) found with tape ${TAPE_SERIAL}." echo -n -e "\033]0;Deck:${SERIAL} LTO:${TAPE_SERIAL}\007" if [[ $(uname -s) = "Darwin" ]] ; then MOUNT_DIR="/Volumes/${TAPE_SERIAL}" From 99f609ed07ad0c61bc56f01234464b5f64c5eaba Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Mon, 12 Jun 2023 14:18:08 -0400 Subject: [PATCH 06/21] more verbosity when loading the tape doesn't work right away --- mountlto | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mountlto b/mountlto index 2b2f199..2617918 100755 --- a/mountlto +++ b/mountlto @@ -127,11 +127,15 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 5 ]]; do ((COUNTER++)) ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}" + LTFS_ERR="${?}" # find barcode by either "Volser(Barcode)" (Quantum style) or "Tape attribute: Barcode" (IBM style) TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1) - echo -n "." + if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then + _report -w "ltfs has an error code ${LTFS_ERR} ${#TAPE_SERIAL} ${TAPE_SERIAL}" + cat "${LOADTMP}" + _report -w "Will retry." + fi done - echo if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then if [[ $(cat "${LOADTMP}" | grep -a "Mounting the volume") && ! $(cat "${LOADTMP}" | grep -a "failed to load the tape") ]] ; then DECKSERIAL=$(grep -a -o "serial number is [A-Z0-9]*" "${LOADTMP}" | awk '{print $4}') From cbe5b4a6e5dd4b221a5656eb3e084c3dec0176c5 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Thu, 15 Jun 2023 11:51:15 -0400 Subject: [PATCH 07/21] change logic for picking a deck Now will use first available deck, else will ask if there's multiple options. --- formatlto | 48 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/formatlto b/formatlto index ec8e862..b248307 100755 --- a/formatlto +++ b/formatlto @@ -95,32 +95,48 @@ else _report -d "Will use compression." fi -if [ ! "${LTO_ARRAY}" ] ; then - if [[ "$(uname -s)" = "Darwin" ]] ; then - # try to figure out how many LTO decks are attached - LTO_ARRAY=($(system_profiler SPSASDataType | grep "SCSI Target Identifier" | cut -d : -f2 | sort | xargs)) +LTO_LIST="$(ltfs -o device_list 2>&1 | grep "Device Name = [0-9]*")" +# try to figure out how many lto decks are attached +LTO_COUNT=$(echo -n "${LTO_LIST}" | grep -c '^') +if [[ "${LTO_COUNT}" -eq 0 ]] ; then + if [[ "$(ltfs -h 2>&1 | grep -c device_list)" -eq 0 ]] ; then + _report -w "The installed version of ltfs at $(which ltfs) doesn't support the '-o device_list' option. Please upgrade ltfs to 2.4.4.0 or newer." + exit 1 else - #### TO DO: itemize when there are multiple LTO drives in Linux and Windows - LTO_ARRAY=(0) + _report -w "No available LTO decks were found or they're all busy." + df -Ph | grep "^Filesystem\|^ltfs" | sort + exit 1 fi fi -if [[ "${#LTO_ARRAY[@]}" -gt 1 ]] ; then - PS3="${BLUE}Which LTO deck?${NC} " - eval set "${LTO_ARRAY[@]}" - select DECK in "$@" ; do - break - done - if [ ! "${DECK}" ] ; then - _report -w "Error: You selected an invalid deck." - exit 1 +_report -d "Checking for ready decks amongst..." +printf "%-12s %20s %20s \n" "Device ID" "Product ID" "Serial Number" +echo "${LTO_LIST}" | while read LTO_RECORD ; do + DEVICE_ID="$(echo "${LTO_RECORD}" | grep -o "Device Name = [^ ,]*" | cut -d = -f2 | sed 's/ //g')" + PRODUCT_ID="$(echo "${LTO_RECORD}" | grep -o "Product ID = [^ ,]*" | cut -d = -f2 | sed 's/ //g')" + SERIAL="$(echo "${LTO_RECORD}" | grep -o "Serial Number = [^ ,]*" | cut -d = -f2 | sed 's/ //g')" + printf "%-12s %20s %20s \n" "${DEVICE_ID}" "${PRODUCT_ID}" "${SERIAL}" +done +echo + +if [[ "${LTO_COUNT}" -eq 1 ]] ; then + DEVICE_ID="$(echo "${LTO_LIST}" | grep -o "Device Name = [^ ,]*" | cut -d = -f2 | sed 's/ //g')" + PRODUCT_ID="$(echo "${LTO_LIST}" | grep -o "Product ID = [^ ,]*" | cut -d = -f2 | sed 's/ //g')" + SERIAL="$(echo "${LTO_LIST}" | grep -o "Serial Number = [^ ,]*" | cut -d = -f2 | sed 's/ //g')" + if [[ "$(uname -s)" = "Linux" ]] ; then + DECK="/dev/sg3" + _report -d "Will use ${DECK} to format the tape." + else + DECK="${DEVICE_ID}" + _report -d "Will use Deck ${DECK} (${SERIAL}) to format the tape." fi else #### TO DO: select the correct path to the drive on Linux and Windows if [[ "$(uname -s)" = "Linux" ]] ; then DECK='/dev/sg3' else - DECK=0 + _report -q -n "Enter the Device ID for the deck to use: " + read DECK fi fi From 8d2f579d2324e194fd41b6c1a865e027e511cc47 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Thu, 15 Jun 2023 11:52:08 -0400 Subject: [PATCH 08/21] add local functions to reduce need for mmfunctions --- collectionchecksum | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/collectionchecksum b/collectionchecksum index 91db014..bac84ce 100755 --- a/collectionchecksum +++ b/collectionchecksum @@ -6,9 +6,33 @@ # presumes a highly specific directory structure based on CUNY TV's Archival # Information Package directory structure. -SCRIPTDIR=$(dirname "${0}") +_get_iso8601(){ + date +%FT%T +} -. "${SCRIPTDIR}/mmfunctions" || { echo "Missing '${SCRIPTDIR}/mmfunctions'. Exiting." ; exit 1 ; } +_report(){ + local RED="$(tput setaf 1)" # Red - For Warnings + local GREEN="$(tput setaf 2)" # Green - For Declarations + local BLUE="$(tput setaf 4)" # Blue - For Questions + local NC="$(tput sgr0)" # No Color + local COLOR="" + local STARTMESSAGE="" + local ECHOOPT="" + OPTIND=1 + while getopts ":qdwstn" OPT; do + case "${OPT}" in + q) COLOR="${BLUE}" ;; # question mode, use color blue + d) COLOR="${GREEN}" ;; # declaration mode, use color green + w) COLOR="${RED}" ;; # warning mode, use color red + s) STARTMESSAGE+=([$(basename "${0}")] ) ;; # prepend scriptname to the message + t) STARTMESSAGE+=($(_get_iso8601) '- ' ) ;; # prepend timestamp to the message + n) ECHOOPT="-n" ;; # to avoid line breaks after echo + esac + done + shift $(( OPTIND - 1 )) + MESSAGE="${1}" + echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[*]}${MESSAGE}${NC}" +} _maketemp(){ mktemp -q "/tmp/$(basename "${0}").XXXXXX" From 1f13b01fa21647de6408ed4e887bec75d4688a43 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Thu, 15 Jun 2023 11:52:29 -0400 Subject: [PATCH 09/21] remove premis db components --- collectionchecksum | 3 --- 1 file changed, 3 deletions(-) diff --git a/collectionchecksum b/collectionchecksum index bac84ce..1de9e99 100755 --- a/collectionchecksum +++ b/collectionchecksum @@ -45,7 +45,6 @@ _maketemp(){ while [ "${*}" != "" ] ; do INPUT="${1}" - _report_to_db TAPECHECKSUMFILE="${INPUT}/tapechecksum.md5" if [ -f "${TAPECHECKSUMFILE}" ] ; then _report -wt "A collection checksum already exists at ${TAPECHECKSUMFILE}" @@ -69,6 +68,4 @@ while [ "${*}" != "" ] ; do sort -k2 "${TEMPCHECKSUMFILE}" | grep -v .DS_Store >> "${TAPECHECKSUMFILE}" shift - _eventoutcome_update - _report_fixity_db done From fcb5e87e6924d57863a4edaf872c15d42dc17dc4 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Thu, 15 Jun 2023 11:52:40 -0400 Subject: [PATCH 10/21] updates to maketemp function --- collectionchecksum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collectionchecksum b/collectionchecksum index 1de9e99..c6a5fff 100755 --- a/collectionchecksum +++ b/collectionchecksum @@ -35,10 +35,10 @@ _report(){ } _maketemp(){ - mktemp -q "/tmp/$(basename "${0}").XXXXXX" + mktemp -t "$(basename "${0}")" if [ "${?}" -ne 0 ] ; then _report -w "${0}: Can't create temp file. Exiting." - _writeerrorlog "_maketemp" "was unable to create the temp file, so the script had to exit." + _report -w "_maketemp was unable to create the temp file, so the script had to exit." exit 1 fi } From 0d7036feb9a98a4e0bf0da74003f0755ac8960f9 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Thu, 15 Jun 2023 11:53:57 -0400 Subject: [PATCH 11/21] refactor method to compile tapechecksum now it's relative to folder and discovers any file that needs a checksum (not covered in checksum.md5) and reports on files missing from collected checksum.md5 --- collectionchecksum | 51 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/collectionchecksum b/collectionchecksum index c6a5fff..40ee1aa 100755 --- a/collectionchecksum +++ b/collectionchecksum @@ -46,26 +46,51 @@ _maketemp(){ while [ "${*}" != "" ] ; do INPUT="${1}" TAPECHECKSUMFILE="${INPUT}/tapechecksum.md5" + TEMP_CHECKSUM_COMPILATION="$(_maketemp)" if [ -f "${TAPECHECKSUMFILE}" ] ; then _report -wt "A collection checksum already exists at ${TAPECHECKSUMFILE}" exit 1 fi - TEMPCHECKSUMFILE=$(_maketemp) - cd "${INPUT}" + _report -d -n "Concatenating existing checksum.md5 files ... " + find "${INPUT}" -type f -name checksum.md5 | while read CHECKSUM_FILE ; do + UPF="$(basename "$(dirname "${CHECKSUM_FILE}")")" + if [[ "${UPF}" == "metadata" ]] ; then + PATH_CLEANING="$(basename "$(dirname "$(dirname "${CHECKSUM_FILE}")")")/objects/" + else + PATH_CLEANING="$(echo "${CHECKSUM_FILE}" | sed "s|${INPUT}/||g;s|checksum.md5||g")" + fi + sed "s| ./| ./${PATH_CLEANING}|g" "${CHECKSUM_FILE}" + done > "${TEMP_CHECKSUM_COMPILATION}" + _report -d "$(wc -l "${TEMP_CHECKSUM_COMPILATION}" | awk '{print $1}') checksum.md5 were gathered for ${INPUT}." - #loop through the directories, find the checksum files, cat them together, and also sed to add the media ids - while read CHECKSUMFILES ; do - PACKAGE="$(basename "$(dirname "$(dirname "${CHECKSUMFILES}")")")" - _report -d "Concatenating ${CHECKSUMFILES}." - cat "${CHECKSUMFILES}" | sed "s| ./| ./${PACKAGE}/objects/|g" >> "${TEMPCHECKSUMFILE}" - done < <(find "${INPUT}" -maxdepth 3 -mindepth 3 -type f -iname "checksum.md5") + if [[ -d "${INPUT}" ]] ; then + cd "${INPUT}" + TEMP_FILE_LIST="$(_maketemp)" + TEMP_LIST2MD5="$(_maketemp)" + find . -type f ! -name ".DS_Store" > "${TEMP_FILE_LIST}" + LIST_COMPARE="$(diff <(cut -c 35- "${TEMP_CHECKSUM_COMPILATION}" | sort) <(sort "${TEMP_FILE_LIST}"))" + MISSING_FILES="$(echo "${LIST_COMPARE}" | grep "^<")" + echo "${LIST_COMPARE}" | grep "^>" | cut -c 3- > "${TEMP_LIST2MD5}" + if [[ -n "${MISSING_FILES}" ]] ; then + _report -w "Warning there are files that are in the checksum.md5 files but not present" + echo "${MISSING_FILES}" + exit 1 + fi + TEMP_NEWMD5S="$(_maketemp)" + if [[ -s "${TEMP_LIST2MD5}" ]] ; then + _report -d "For the tape checksum, these $(wc -l "${TEMP_LIST2MD5}" | awk '{print $1}') additional files must be checksummed" + cat "${TEMP_LIST2MD5}" + md5deep -rel -f "${TEMP_LIST2MD5}" > "${TEMP_NEWMD5S}" + _report -d "Checksummed $(wc -l "${TEMP_CHECKSUM_COMPILATION}" | awk '{print $1}') additional files." + fi + cat "${TEMP_NEWMD5S}" "${TEMP_CHECKSUM_COMPILATION}" | sort -k2 > "${TAPECHECKSUMFILE}" + _report -d "${TAPECHECKSUMFILE} is ready with $(wc -l "${TAPECHECKSUMFILE}" | awk '{print $1}') checksums." + else + _report -d "${INPUT} is not a directory. Exiting." + exit 1 + fi - #create md5 checksums for all of the files in the metadata directory - _report -d "Creating checksums for all metadata directories." - md5deep -rel ./*/metadata >> "${TEMPCHECKSUMFILE}" - _report -d "Sorting and saving ${TAPECHECKSUMFILE}." - sort -k2 "${TEMPCHECKSUMFILE}" | grep -v .DS_Store >> "${TAPECHECKSUMFILE}" shift done From 67b948bac897bd08e5810e28aa53124479a73ccd Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Thu, 15 Jun 2023 11:54:08 -0400 Subject: [PATCH 12/21] add a user log to collectionchecksum --- collectionchecksum | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/collectionchecksum b/collectionchecksum index 40ee1aa..ab23783 100755 --- a/collectionchecksum +++ b/collectionchecksum @@ -46,6 +46,7 @@ _maketemp(){ while [ "${*}" != "" ] ; do INPUT="${1}" TAPECHECKSUMFILE="${INPUT}/tapechecksum.md5" + TAPEWRITELOG="${INPUT}/tapewritelog.txt" TEMP_CHECKSUM_COMPILATION="$(_maketemp)" if [ -f "${TAPECHECKSUMFILE}" ] ; then _report -wt "A collection checksum already exists at ${TAPECHECKSUMFILE}" @@ -90,7 +91,14 @@ while [ "${*}" != "" ] ; do exit 1 fi + _report -q -n "Name of operator: " + read OP + _report -q -n "Note about tape write: " + read NOTE + echo "operator: ${OP}" > "${TAPEWRITELOG}" + echo "datetime: $(_get_iso8601)" >> "${TAPEWRITELOG}" + echo "note: ${NOTE}" >> "${TAPEWRITELOG}" shift done From f5081387eb0f5ff2e882aefd6ff59e0c19d3c4ab Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 13:44:37 -0400 Subject: [PATCH 13/21] add functions to drop need for mmfunctions --- writelto | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/writelto b/writelto index f9b49f8..b6c9305 100755 --- a/writelto +++ b/writelto @@ -5,8 +5,7 @@ # LTFS-formated LTO tape. SCRIPTDIR=$(dirname "${0}") -. "${SCRIPTDIR}/mmfunctions" || { echo "Missing '${SCRIPTDIR}/mmfunctions'. Exiting." ; exit 1 ; } -DEPENDENCIES=(gcp mmfunctions) +DEPENDENCIES=(rsync gcp) TAPE_MOUNT_POINT="/Volumes" TAPE_SERIAL_REGEX="^[A-Z0-9]{6}$" unset HIDDEN_FILES @@ -21,6 +20,106 @@ if [[ "$(uname -s)" = "Darwin" ]] ; then fi fi +_check_dependencies(){ + DEPS_OK=YES + while [ "${*}" != "" ] ; do + DEPENDENCY="${1}" + if [ ! "$(which "${DEPENDENCY}")" ] ; then + _report -wt "This script requires ${DEPENDENCY} to run but it is not installed" + _report -wt "If you are running ubuntu or debian you might be able to install ${DEPENDENCY} with the following command" + _report -wt "sudo apt-get install ${DEPENDENCY}" + _report -wt "If you are running mac you might be able to install ${DEPENDENCY} with the following command" + _report -wt "brew install ${DEPENDENCY}" + DEPS_OK=NO + fi + shift + done + if [[ "${DEPS_OK}" = "NO" ]]; then + _report -wt "Unmet dependencies" + _report -wt "Aborting!" + exit 1 + else + return 0 + fi +} + +_check_dir(){ + if [[ ! -d "${1}" ]] ; then + _report -w "[${1}] is not a directory." + _usage + exit 1 + fi +} + +_check_for_colons(){ + COLONALERT=$(find "${1}" -iname '*:*') + if [ -n "${COLONALERT}" ] ; then + _report -w "Note that LTFS formatted tapes called store files with colons in their filenames. Colons have been detected in the following file(s):" + for i in ${COLONALERT} ; do + _report -w " ${i}" + done + _report -w "Exiting" + exit 1 + fi +} + +_check_for_lto_md5_flags(){ + if [ -z "${LTO_MD5_FLAGS}" ] ; then + LTO_MD5_FLAGS="md5deep -rel" + fi +} + +_check_for_lto_index_dir(){ + # if LTO_INDEX_DIR is set in ltopers.conf then that will be used, otherwise a default + if [ -z "${LTO_INDEX_DIR}" ] ; then + LTO_INDEX_DIR="${DEFAULT_LTO_INDEX_DIR}" + fi +} + +_get_iso8601_c(){ + date +%Y%m%d-%H%M%S +} + +_mkdir2(){ + local DIR2MAKE="" + while [ "${*}" != "" ] ; do + DIR2MAKE="${1}" + if [ ! -d "${DIR2MAKE}" ] ; then + mkdir -p "${DIR2MAKE}" + if [ "${?}" -ne 0 ]; then + _report -wt "${0}: Can't create directory at ${DIR2MAKE}" + exit 1 + fi + fi + shift + done +} + +_report(){ + local RED="$(tput setaf 1)" # Red - For Warnings + local GREEN="$(tput setaf 2)" # Green - For Declarations + local BLUE="$(tput setaf 4)" # Blue - For Questions + local NC="$(tput sgr0)" # No Color + local COLOR="" + local STARTMESSAGE="" + local ECHOOPT="" + local LOG_MESSAGE="" + OPTIND=1 + while getopts ":qdwstn" OPT; do + case "${OPT}" in + q) COLOR="${BLUE}" ;; # question mode, use color blue + d) COLOR="${GREEN}" ;; # declaration mode, use color green + w) COLOR="${RED}" ;; # warning mode, use color red + s) STARTMESSAGE+=([$(basename "${0}")] ) ;; # prepend scriptname to the message + t) STARTMESSAGE+=($(_get_iso8601_c) '- ' ) ;; # prepend timestamp to the message + n) ECHOOPT="-n" ;; # to avoid line breaks after echo + esac + done + shift $(( OPTIND - 1 )) + MESSAGE="${1}" + echo "${ECHOOPT}" "${COLOR}${STARTMESSAGE[*]}${MESSAGE}${NC}" +} + _usage(){ cat < Date: Wed, 16 Aug 2023 13:45:00 -0400 Subject: [PATCH 14/21] drop premis db part --- writelto | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/writelto b/writelto index b6c9305..ca9c0b2 100755 --- a/writelto +++ b/writelto @@ -233,20 +233,3 @@ case "${TAPE_EJECT}" in y|Y) umount "${TAPE_PATH}" ;; *) echo "Done writing but not ejecting ${TAPE_SERIAL}." ;; esac - -if [[ "${PREMIS_DB}" = "Y" ]] ; then - _eventoutcome_update - _report_fixity_db - _report_schema_db - _db_email_delivery "${SQL_ERROR_EMAIL_TO}" - if [[ "${VERIFY}" = "Y" ]] ; then - VERIFY_CHECK="Y" - unset db_fixity - db_fixity=$(cat "${READBACKDIR}/${TAPE_SERIAL}_ReadBack_checksum_${VERIFYTIME}.md5") - if [ -z "${db_fixity}" ] ; then - event_outcome="No fixity information" - fi - _report_fixity_db - _db_email_delivery "${SQL_ERROR_EMAIL_TO}" - fi -fi From 6dbdda9cfb5453413513efd73a5fb001676544d7 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 13:45:26 -0400 Subject: [PATCH 15/21] update formatting of usage documentation --- writelto | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/writelto b/writelto index ca9c0b2..073ac82 100755 --- a/writelto +++ b/writelto @@ -128,14 +128,14 @@ LTFS-formated LTO tape. Dependencies: ${DEPENDENCIES[@]} Usage: $(basename "${0}") -t [-e] [-v] | -h - -t tape serial - -e state yes (Y) or no (N) to ejecting the tape after write, default - is yes - -v reads back and creates checksums for the contents of a tape, and - writes checksums to a file named with the tape name and date, - located in the LTO logs directory - -x use barcode label as tape serial (caveat: this is not standard) - -h display this help + -t tape serial + -e [Y|N] state yes (Y) or no (N) to ejecting the tape after write, default + is yes + -v reads back and creates checksums for the contents of a tape, and + writes checksums to a file named with the tape name and date, + located in the LTO logs directory + -x use barcode label as tape serial (caveat: this is not standard) + -h display this help EOF } From 665ab53b8c49f439c1ca4af735dbf805c68541a3 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 13:45:49 -0400 Subject: [PATCH 16/21] move a few things to later after functions are defined --- writelto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/writelto b/writelto index 073ac82..7e8d3d4 100755 --- a/writelto +++ b/writelto @@ -10,8 +10,6 @@ TAPE_MOUNT_POINT="/Volumes" TAPE_SERIAL_REGEX="^[A-Z0-9]{6}$" unset HIDDEN_FILES TAPE_EJECT="Y" -_check_for_lto_index_dir -LTO_LOGS="${LTO_INDEX_DIR}" # no .DS_Store onto external storage if [[ "$(uname -s)" = "Darwin" ]] ; then @@ -155,6 +153,8 @@ shift "$((OPTIND-1))" SOURCE_DIR="${1}" +_check_for_lto_index_dir +LTO_LOGS="${LTO_INDEX_DIR}" if [[ ! "${TAPE_SERIAL}" ]] ; then _report -qn "Enter the LTO tape barcode: " read -e TAPE_SERIAL From 8f190bf7f6718e87164607abaa58214869bc35fe Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 13:46:36 -0400 Subject: [PATCH 17/21] fix function name --- writelto | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/writelto b/writelto index 7e8d3d4..c6f1157 100755 --- a/writelto +++ b/writelto @@ -155,6 +155,8 @@ SOURCE_DIR="${1}" _check_for_lto_index_dir LTO_LOGS="${LTO_INDEX_DIR}" +_check_dependencies "${DEPENDENCIES[@]}" + if [[ ! "${TAPE_SERIAL}" ]] ; then _report -qn "Enter the LTO tape barcode: " read -e TAPE_SERIAL @@ -174,14 +176,13 @@ _check_for_colons "${SOURCE_DIR}" TAPE_PATH="${TAPE_MOUNT_POINT}/${TAPE_SERIAL}" -_checkdir "${SOURCE_DIR}" # remove .DS_Store files find "${SOURCE_DIR}/" -name '.DS_Store' -type f -delete +_check_dir "${SOURCE_DIR}" if [ -f "${SOURCE_DIR}"/tapechecksum.md5 ] ; then db_fixity=$(cat "${SOURCE_DIR}"/tapechecksum.md5) fi -_checkdir "${TAPE_PATH}" gcp --preserve=mode,timestamps -nRv "${SOURCE_DIR}/"* "${TAPE_PATH}" @@ -194,6 +195,7 @@ if [[ "${HIDDEN_FILES}" ]] ; then "${SCRIPTDIR}/migratefiles" -o "${TAPE_PATH}" "${SOURCE_DIR}/" RSYNC_ERR_2="$?" fi +_check_dir "${TAPE_PATH}" WRITELTODIR="${LTO_LOGS}/writelto" _mkdir2 "${WRITELTODIR}" From 1c42481c7a62c69640480d8c4af94813bf026559 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 16:09:47 -0400 Subject: [PATCH 18/21] add new pattern to find tape serial --- mountlto | 2 +- writelto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mountlto b/mountlto index 2617918..606d376 100755 --- a/mountlto +++ b/mountlto @@ -129,12 +129,12 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}" LTFS_ERR="${?}" # find barcode by either "Volser(Barcode)" (Quantum style) or "Tape attribute: Barcode" (IBM style) - TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1) if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then _report -w "ltfs has an error code ${LTFS_ERR} ${#TAPE_SERIAL} ${TAPE_SERIAL}" cat "${LOADTMP}" _report -w "Will retry." fi + TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*\|Volume mounted successfully*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1) done if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then if [[ $(cat "${LOADTMP}" | grep -a "Mounting the volume") && ! $(cat "${LOADTMP}" | grep -a "failed to load the tape") ]] ; then diff --git a/writelto b/writelto index c6f1157..a7a9241 100755 --- a/writelto +++ b/writelto @@ -212,7 +212,7 @@ fi _check_for_lto_md5_flags if [[ "${VERIFY}" = "Y" ]] ; then - VERIFYTIME=$(_get_iso8601_c) + VERIFYTIME="$(_get_iso8601_c)" READBACKDIR="${LTO_LOGS}/readback_checksums" _mkdir2 "${READBACKDIR}" find "${TAPE_PATH}" -type f ! -name .DS_Store -exec $LTO_MD5_FLAGS "{}" >> "${READBACKDIR}/${TAPE_SERIAL}_ReadBack_checksum_${VERIFYTIME}.md5" \; From 8b18885481cad13633968799ef59242668142ee2 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 16:10:01 -0400 Subject: [PATCH 19/21] skip handing an error --- mountlto | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mountlto b/mountlto index 606d376..efb3a31 100755 --- a/mountlto +++ b/mountlto @@ -129,11 +129,6 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}" LTFS_ERR="${?}" # find barcode by either "Volser(Barcode)" (Quantum style) or "Tape attribute: Barcode" (IBM style) - if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then - _report -w "ltfs has an error code ${LTFS_ERR} ${#TAPE_SERIAL} ${TAPE_SERIAL}" - cat "${LOADTMP}" - _report -w "Will retry." - fi TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*\|Volume mounted successfully*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1) done if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then From f60c6e6e02e2a16a3c958cfbef3dd4ab2a08180d Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 16:11:04 -0400 Subject: [PATCH 20/21] adjust handling of line break when trying to check --- mountlto | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mountlto b/mountlto index efb3a31..d6f62f7 100755 --- a/mountlto +++ b/mountlto @@ -123,14 +123,16 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do LOADTMP=$(_maketemp "${SERIAL}") unset TAPE_SERIAL COUNTER=0 - _report -d "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})" while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 5 ]]; do + _report -dn "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})" ((COUNTER++)) ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}" LTFS_ERR="${?}" # find barcode by either "Volser(Barcode)" (Quantum style) or "Tape attribute: Barcode" (IBM style) TAPE_SERIAL=$(grep "Volser(Barcode).*\|Tape attribute: Medium Label*\|Tape attribute: Barcode.*\|Volume mounted successfully*" "${LOADTMP}" | grep -Eo "\b[A-Z0-9]{6}\b|\b[A-Z0-9]{8}\b" | head -n 1) + echo -n "." done + echo if [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] ; then if [[ $(cat "${LOADTMP}" | grep -a "Mounting the volume") && ! $(cat "${LOADTMP}" | grep -a "failed to load the tape") ]] ; then DECKSERIAL=$(grep -a -o "serial number is [A-Z0-9]*" "${LOADTMP}" | awk '{print $4}') From fd4297dde1a5d2de3fadfe98af1da3c02e11a88f Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 16 Aug 2023 16:11:15 -0400 Subject: [PATCH 21/21] reduce trial and error down to 2 --- mountlto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mountlto b/mountlto index d6f62f7..60fe3d4 100755 --- a/mountlto +++ b/mountlto @@ -123,8 +123,8 @@ echo "${LTO_LIST}" | while read LTO_RECORD ; do LOADTMP=$(_maketemp "${SERIAL}") unset TAPE_SERIAL COUNTER=0 - while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 5 ]]; do _report -dn "Checking for tape barcode with deck ${DEVICE_ID} (${SERIAL})" + while [[ "${#TAPE_SERIAL}" -lt "${MINIMUM_TAPE_SERIAL_LENGTH}" ]] && [[ "${COUNTER}" -lt 2 ]]; do ((COUNTER++)) ltfs -f -o devname="${DEVICE_ID}" 2> "${LOADTMP}" LTFS_ERR="${?}"