From bfe38461eabce7eaf1ce51c0db80dc4b19bd87c8 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 00:09:17 -0500 Subject: [PATCH 01/15] add apprise and ntfy support for notification --- .../snapraid/files/etc-snapraid-diff_conf.j2 | 5 + usr/sbin/omv-snapraid-diff | 568 ++++++++++-------- .../confdb/create.d/conf.service.snapraid.sh | 4 + .../datamodels/conf.service.snapraid.json | 14 + .../datamodels/rpc.snapraid.json | 16 + ...-services-snapraid-settings-form-page.yaml | 30 + 6 files changed, 395 insertions(+), 242 deletions(-) diff --git a/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 b/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 index a09d0d3..6b2d6ed 100644 --- a/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 +++ b/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 @@ -6,3 +6,8 @@ SCRUB_FREQUENCY_IN_DAYS={{ config.scrubfreq }} UPD_THRESHOLD={{ config.updthreshold }} DEL_THRESHOLD={{ config.delthreshold }} SCRUB_PERCENT={{ config.scrubpercent }} +{{ 'USE_APPRISE=true' if config.apprise | to_bool else 'USE_APPRISE=false' }} +{{ 'USE_NTFY=true' if config.ntfy | to_bool else 'USE_NTFY=false' }} +APPRISE_CMD={{ config.apprisecmd }} +NTFY_CMD={{ config.ntfycmd }} + diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 52735d8..1e98107 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -52,7 +52,6 @@ declare -i UPDATE_COUNT=0 declare -i PREHASH_VALUE=0 - ARRAY_CONF="${1}" # Set path to snapraid.conf @@ -75,43 +74,128 @@ SNAPRAID_LOG="/var/log/snapraid.log" ################################################################ # -# name: _send_email -# parameter : # $1 = Subject - # $2 = file with mailtext - # Optional: - # $3 = Who sent this message (full name), e.g. "Server Status-Report" - # $4 = Answer to this adress -# return: 0 = All OK -# 1 = Error happend +# name: _send_notification +# parameter : +# $1 = Subject +# $2 = file with mailtext +# Optional: +# $3 = Who sent this message (full name), e.g. "Server Status-Report" +# $4 = Answer to this address +# return: 0 = All OK +# 1 = Error happened # -_send_email(){ -## the below does not work with the internal OMV-Settings -# mail -s "$1" -a 'Content-Type: text/plain; charset=utf-8' $MAIL -- -F "$3" -f "$4" < $2 - -if [ $SENDMAIL = "false" ]; then - _log "INFO: Mail should be sent here - SENDMAIL set to false" - echo "Mail should be sent here - SENDMAIL set to false" - return 0 -fi +_send_notification() { + + MAIL_FAIL=false + APPRISE_FAIL=false + NTFY_FAIL=false -# add mail footer -cat >> $TMP_OUTPUT <>$TMP_OUTPUT </dev/null 2>&1; then + return 0 + else + return 1 + fi +} ################################################################ # # name : _log @@ -119,18 +203,18 @@ mail -c "$OMV_MAIL_secondarymail" -s "$1" "$OMV_MAIL_primarymail" < $2 # return : none # _log() { - [[ "$*" =~ ^([A-Za-z]*):\ (.*) ]] && - { - PRIORITY=${BASH_REMATCH[1]} # INFO, DEBUG, WARN - LOGMESSAGE=${BASH_REMATCH[2]} # the Log-Message - } - - if $DEBUG ; then - echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" - echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >> $SNAPRAID_LOG - else - echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >> $SNAPRAID_LOG - fi + [[ "$*" =~ ^([A-Za-z]*):\ (.*) ]] && + { + PRIORITY=${BASH_REMATCH[1]} # INFO, DEBUG, WARN + LOGMESSAGE=${BASH_REMATCH[2]} # the Log-Message + } + + if $DEBUG; then + echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" + echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG + else + echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG + fi } ################################################################ @@ -141,8 +225,8 @@ _log() { # # removes $TMP_OUTPUT _rmtmp() { - rm $TMP_OUTPUT >/dev/null 2>&1 - rm $TMP_DIR >/dev/null 2>&1 + rm $TMP_OUTPUT >/dev/null 2>&1 + rm $TMP_DIR >/dev/null 2>&1 } ################################################################ @@ -153,11 +237,11 @@ _rmtmp() { # : 1 = last command ended with an error -> [ $? -gt 1 ] # _check_success() { - if [ $? = 0 ]; then - return 0 - else - return 1 - fi + if [ $? = 0 ]; then + return 0 + else + return 1 + fi } ################################################################ @@ -168,7 +252,7 @@ _check_success() { # adds mail text # _add_mail_header() { - cat > $TMP_OUTPUT <$TMP_OUTPUT < $TMP_SCRUB_DIR/snapraid_scrub_counter - COUNTER=1 + echo "1" >$TMP_SCRUB_DIR/snapraid_scrub_counter + COUNTER=1 fi # enable only one of the two below!!! let COUNTER++ @@ -305,14 +390,14 @@ let COUNTER++ # define Variables, if not set in snapraid-diff if [ -z ${SCRUB_PERCENT} ]; then - SCRUB_PERCENT=100 - _log "INFO: SCRUB_PERCENT not set in '$SR_DIFF_CONF'" - _log "INFO: set it to $SCRUB_PERCENT" + SCRUB_PERCENT=100 + _log "INFO: SCRUB_PERCENT not set in '$SR_DIFF_CONF'" + _log "INFO: set it to $SCRUB_PERCENT" fi if [ -z ${SCRUB_OLDER_THAN_DAYS} ]; then - SCRUB_OLDER_THAN_DAYS=0 - _log "INFO: SCRUB_OLDER_THAN_DAYS not set in '$SR_DIFF_CONF'" - _log "INFO: set it to $SCRUB_OLDER_THAN_DAYS" + SCRUB_OLDER_THAN_DAYS=0 + _log "INFO: SCRUB_OLDER_THAN_DAYS not set in '$SR_DIFF_CONF'" + _log "INFO: set it to $SCRUB_OLDER_THAN_DAYS" fi ### End Scrub Section ### @@ -327,45 +412,44 @@ _log "INFO: Checking 'disks' in '$SNAPRAID_CONF'" ### check if all hdds in the config are there for OMV_DISK in $OMV_SNAPRAID_DISKS; do - if $DEBUG; then _log "DEBUG: found: OMV_DISK: $OMV_DISK"; fi - - if ! grep $OMV_DISK /etc/mtab > /dev/null 2>&1; then - - #if [ ! -b $OMV_DISK ]; then # doesn't work - no block device found :o - _log "WARN: ERROR: Disk $OMV_DISK was not found or is not mounted!" - _log "WARN: Sync job cancelled to prevent data loss." - _log "WARN: Please check your disks!" - _log "WARN: Replace damaged Disk and proceed with 'snapraid fix'" - _log "WARN: to rebuild the raid array and recover your data." - _log "WARN: Then, use 'snapraid check' to verify the integrity of the recovered files." - _log "WARN: Lastly, use 'snapraid sync' to update the parity files." - - echo "ERROR - Data Disk $OMV_DISK not found or is not mounted!" >> $TMP_OUTPUT - echo "Sync job cancelled to prevent data loss." >> $TMP_OUTPUT - echo " " >> $TMP_OUTPUT - echo "Please check your disks!" >> $TMP_OUTPUT - echo "Replace damaged Disk and proceed with 'snapraid fix'" >> $TMP_OUTPUT - echo "to rebuild the raid array and recover your data." >> $TMP_OUTPUT - echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >> $TMP_OUTPUT - echo "Lastly, use 'snapraid sync' to update the parity files." >> $TMP_OUTPUT - - _send_email "$EMAIL_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" - _check_success && _rmtmp - _log "WARN: script exits here due to missing disk ..." - exit 1; - fi + if $DEBUG; then _log "DEBUG: found: OMV_DISK: $OMV_DISK"; fi + + if ! grep $OMV_DISK /etc/mtab >/dev/null 2>&1; then + + #if [ ! -b $OMV_DISK ]; then # doesn't work - no block device found :o + _log "WARN: ERROR: Disk $OMV_DISK was not found or is not mounted!" + _log "WARN: Sync job cancelled to prevent data loss." + _log "WARN: Please check your disks!" + _log "WARN: Replace damaged Disk and proceed with 'snapraid fix'" + _log "WARN: to rebuild the raid array and recover your data." + _log "WARN: Then, use 'snapraid check' to verify the integrity of the recovered files." + _log "WARN: Lastly, use 'snapraid sync' to update the parity files." + + echo "ERROR - Data Disk $OMV_DISK not found or is not mounted!" >>$TMP_OUTPUT + echo "Sync job cancelled to prevent data loss." >>$TMP_OUTPUT + echo " " >>$TMP_OUTPUT + echo "Please check your disks!" >>$TMP_OUTPUT + echo "Replace damaged Disk and proceed with 'snapraid fix'" >>$TMP_OUTPUT + echo "to rebuild the raid array and recover your data." >>$TMP_OUTPUT + echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >>$TMP_OUTPUT + echo "Lastly, use 'snapraid sync' to update the parity files." >>$TMP_OUTPUT + + _send_notification "$EMAIL_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _check_success && _rmtmp + _log "WARN: script exits here due to missing disk ..." + exit 1 + fi done ################################################################ # start snapraid DIFF command _log "INFO: SnapRAID DIFF started" -echo "SnapRAID DIFF started - $(date)" >> $TMP_OUTPUT +echo "SnapRAID DIFF started - $(date)" >>$TMP_OUTPUT # run diff: # output: stdout: WARNINGS only (filtering 'Ignoring mount point') and $TMP_OUTPUT (all WARNINGS) -/usr/bin/snapraid diff --conf ${SNAPRAID_CONF} 2>&1 | tee -a $TMP_OUTPUT | grep WARNING | grep -v 'WARNING! Ignoring mount point' | while read line -do - _log "WARN: $line" +/usr/bin/snapraid diff --conf ${SNAPRAID_CONF} 2>&1 | tee -a $TMP_OUTPUT | grep WARNING | grep -v 'WARNING! Ignoring mount point' | while read line; do + _log "WARN: $line" done # wait for the above cmd to finish @@ -373,8 +457,8 @@ wait _log "INFO: SnapRAID DIFF finished" _log "INFO: ----------------------------------------" -echo "SnapRAID DIFF finished - $(date)" >> $TMP_OUTPUT -echo "----------------------------------------" >> $TMP_OUTPUT +echo "SnapRAID DIFF finished - $(date)" >>$TMP_OUTPUT +echo "----------------------------------------" >>$TMP_OUTPUT # read the values of deleted, added, etc. files since the last sync DEL_COUNT=$(grep -w '^ \{1,\}[0-9]* removed$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) @@ -389,133 +473,133 @@ _log "INFO: Added: [$ADD_COUNT] - Deleted: [$DEL_COUNT] - Moved: [$MOVE_COUNT] - # set pre-hash option PREHASH_VALUE=$(omv_config_get "/config/services/snapraid/prehash") if $DEBUG; then - _log "DEBUG: Pre-Hash option: ${PREHASH_VALUE}" + _log "DEBUG: Pre-Hash option: ${PREHASH_VALUE}" fi # check if files have changed if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $COPY_COUNT -gt 0 ] || [ $UPDATE_COUNT -gt 0 ]; then - UPD_THRESHOLD=$(omv_config_get "/config/services/snapraid/updthreshold") - DEL_THRESHOLD=$(omv_config_get "/config/services/snapraid/delthreshold") - - # New Logic for v 1.2.0 - check whether update threshold is enabled or not - if [ $UPD_THRESHOLD -gt 0 ]; then - # Update threshold is enabled; check if count exceeds threshold - if [ $UPDATE_COUNT -gt $UPD_THRESHOLD ]; then - # Too many files updated; inform user and don't proceed with sync - _log "INFO: Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." - _log "INFO: Please run sync manually if this is not an error condition." - echo "Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." >> $TMP_OUTPUT - echo "Please run sync manually if this is not an error condition" >> $TMP_OUTPUT - - # Increment check variable - UPD_DEL_CHECK=$((UPD_DEL_CHECK+1)) - else - # Update threshold is set but not exceeded. Inform user that UPDATE_COUNT is below threshold and allow to proceed - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." >> $TMP_OUTPUT - fi - else - # UPD_THRESHOLD is disabled, so we reflect this information in email and log - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." >> $TMP_OUTPUT - fi - - # New Logic for v 1.2.0 - check whether delete threshold is enabled or not - if [ $DEL_THRESHOLD -gt 0 ]; then - # Delete threshold is enabled; check if count exceeds threshold - if [ $DEL_COUNT -gt $DEL_THRESHOLD ]; then - # Too many files deleted; inform user and don't proceed with sync - _log "INFO: Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." - _log "INFO: Please run sync manually if this is not an error condition." - echo "Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." >> $TMP_OUTPUT - echo "Please run sync manually if this is not an error condition" >> $TMP_OUTPUT - - # Increment check variable - UPD_DEL_CHECK=$((UPD_DEL_CHECK+2)) - else - # Delete threshold is set but not exceeded. Inform user that DEL_COUNT is below threshold and allow to proceed - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." >> $TMP_OUTPUT - fi - else - # DEL_THRESHOLD is disabled, so we reflect this information in email and log - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." >> $TMP_OUTPUT - fi - - # Proceed with sync if no check errors - if [ $UPD_DEL_CHECK -eq 0 ]; then - # start snapraid SYNC - _log "INFO: " - _log "INFO: SnapRAID SYNC Job started" - _log "INFO: ----------------------------------------" - echo "" >> $TMP_OUTPUT - echo "SnapRAID SYNC Job started - $(date)" >> $TMP_OUTPUT - echo "----------------------------------------" >> $TMP_OUTPUT - - # we should eventually grep % - if [ ${PREHASH_VALUE} -eq 1 ]; then - /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} --pre-hash >> $TMP_OUTPUT - else - /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} >> $TMP_OUTPUT - fi - #wait for the job to finish - wait - - _log "INFO: ----------------------------------------" - _log "INFO: SnapRAID SYNC Job finished" - echo "SnapRAID SYNC Job finished - $(date)" >> $TMP_OUTPUT - echo "----------------------------------------" >> $TMP_OUTPUT - - # start snapraid SCRUB - if [ $RUN_SCRUB = "true" ]; then - if [ $COUNTER -lt $SCRUB_FREQUENCY_IN_DAYS ]; then - _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($COUNTER). No scrub was run." - echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($COUNTER). No scrub was run. - $(date)" >> $TMP_OUTPUT - echo "$COUNTER" > $TMP_SCRUB_DIR/snapraid_scrub_counter - else - _log "INFO: SnapRAID SCRUB Job started" - _log "INFO: ----------------------------------------" - echo "SnapRAID SCRUB Job started - $(date)" >> $TMP_OUTPUT - echo "----------------------------------------" >> $TMP_OUTPUT - /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${SCRUB_OLDER_THAN_DAYS} >> $TMP_OUTPUT - - # wait for the job to finish - wait - - # Reset counter - echo "1" > $TMP_SCRUB_DIR/snapraid_scrub_counter - - _log "INFO: ----------------------------------------" - _log "INFO: SnapRAID SCRUB Job finished" - echo "SnapRAID SCRUB Job finished - $(date)" >> $TMP_OUTPUT - echo "----------------------------------------" >> $TMP_OUTPUT - fi - else - echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >> $TMP_OUTPUT - _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." - fi - - # Send success email - _send_email "$EMAIL_SUBJECT_PREFIX - Sync/Scrub Job COMPLETED" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" - - # If update threshold exceeded - elif [ $UPD_DEL_CHECK -eq 1 ]; then - _send_email "$EMAIL_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" - - # If delete threshold exceeded - elif [ $UPD_DEL_CHECK -eq 2 ]; then - _send_email "$EMAIL_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" - - # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) - else - _send_email "$EMAIL_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" - fi - _check_success && _rmtmp + UPD_THRESHOLD=$(omv_config_get "/config/services/snapraid/updthreshold") + DEL_THRESHOLD=$(omv_config_get "/config/services/snapraid/delthreshold") + + # New Logic for v 1.2.0 - check whether update threshold is enabled or not + if [ $UPD_THRESHOLD -gt 0 ]; then + # Update threshold is enabled; check if count exceeds threshold + if [ $UPDATE_COUNT -gt $UPD_THRESHOLD ]; then + # Too many files updated; inform user and don't proceed with sync + _log "INFO: Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." + _log "INFO: Please run sync manually if this is not an error condition." + echo "Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." >>$TMP_OUTPUT + echo "Please run sync manually if this is not an error condition" >>$TMP_OUTPUT + + # Increment check variable + UPD_DEL_CHECK=$((UPD_DEL_CHECK + 1)) + else + # Update threshold is set but not exceeded. Inform user that UPDATE_COUNT is below threshold and allow to proceed + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." >>$TMP_OUTPUT + fi + else + # UPD_THRESHOLD is disabled, so we reflect this information in email and log + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." >>$TMP_OUTPUT + fi + + # New Logic for v 1.2.0 - check whether delete threshold is enabled or not + if [ $DEL_THRESHOLD -gt 0 ]; then + # Delete threshold is enabled; check if count exceeds threshold + if [ $DEL_COUNT -gt $DEL_THRESHOLD ]; then + # Too many files deleted; inform user and don't proceed with sync + _log "INFO: Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." + _log "INFO: Please run sync manually if this is not an error condition." + echo "Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." >>$TMP_OUTPUT + echo "Please run sync manually if this is not an error condition" >>$TMP_OUTPUT + + # Increment check variable + UPD_DEL_CHECK=$((UPD_DEL_CHECK + 2)) + else + # Delete threshold is set but not exceeded. Inform user that DEL_COUNT is below threshold and allow to proceed + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." >>$TMP_OUTPUT + fi + else + # DEL_THRESHOLD is disabled, so we reflect this information in email and log + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." >>$TMP_OUTPUT + fi + + # Proceed with sync if no check errors + if [ $UPD_DEL_CHECK -eq 0 ]; then + # start snapraid SYNC + _log "INFO: " + _log "INFO: SnapRAID SYNC Job started" + _log "INFO: ----------------------------------------" + echo "" >>$TMP_OUTPUT + echo "SnapRAID SYNC Job started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + + # we should eventually grep % + if [ ${PREHASH_VALUE} -eq 1 ]; then + /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} --pre-hash >>$TMP_OUTPUT + else + /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} >>$TMP_OUTPUT + fi + #wait for the job to finish + wait + + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SYNC Job finished" + echo "SnapRAID SYNC Job finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + + # start snapraid SCRUB + if [ $RUN_SCRUB = "true" ]; then + if [ $COUNTER -lt $SCRUB_FREQUENCY_IN_DAYS ]; then + _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($COUNTER). No scrub was run." + echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($COUNTER). No scrub was run. - $(date)" >>$TMP_OUTPUT + echo "$COUNTER" >$TMP_SCRUB_DIR/snapraid_scrub_counter + else + _log "INFO: SnapRAID SCRUB Job started" + _log "INFO: ----------------------------------------" + echo "SnapRAID SCRUB Job started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${SCRUB_OLDER_THAN_DAYS} >>$TMP_OUTPUT + + # wait for the job to finish + wait + + # Reset counter + echo "1" >$TMP_SCRUB_DIR/snapraid_scrub_counter + + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SCRUB Job finished" + echo "SnapRAID SCRUB Job finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + fi + else + echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT + _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." + fi + + # Send success email + _send_notification "$EMAIL_SUBJECT_PREFIX - Sync/Scrub Job COMPLETED" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + + # If update threshold exceeded + elif [ $UPD_DEL_CHECK -eq 1 ]; then + _send_notification "$EMAIL_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + + # If delete threshold exceeded + elif [ $UPD_DEL_CHECK -eq 2 ]; then + _send_notification "$EMAIL_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + + # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) + else + _send_notification "$EMAIL_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + fi + _check_success && _rmtmp else - # NO, so lets log it and exit - _log "INFO: No change detected. Nothing to do" + # NO, so lets log it and exit + _log "INFO: No change detected. Nothing to do" fi _log "INFO: SnapRAID Job ended." diff --git a/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh b/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh index 7a6ccf8..8b2b55a 100755 --- a/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh +++ b/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh @@ -35,6 +35,10 @@ if ! omv_config_exists "${SERVICE_XPATH}"; then omv_config_add_key "${SERVICE_XPATH}" "nohidden" "0" omv_config_add_key "${SERVICE_XPATH}" "debug" "0" omv_config_add_key "${SERVICE_XPATH}" "sendmail" "1" + omv_config_add_key "${SERVICE_XPATH}" "apprise" "0" + omv_config_add_key "${SERVICE_XPATH}" "apprisecmd" "" + omv_config_add_key "${SERVICE_XPATH}" "ntfy" "0" + omv_config_add_key "${SERVICE_XPATH}" "ntfycmd" "" omv_config_add_key "${SERVICE_XPATH}" "runscrub" "1" omv_config_add_key "${SERVICE_XPATH}" "scrubfreq" "7" omv_config_add_key "${SERVICE_XPATH}" "updthreshold" "0" diff --git a/usr/share/openmediavault/datamodels/conf.service.snapraid.json b/usr/share/openmediavault/datamodels/conf.service.snapraid.json index 587bc70..f4e40ed 100644 --- a/usr/share/openmediavault/datamodels/conf.service.snapraid.json +++ b/usr/share/openmediavault/datamodels/conf.service.snapraid.json @@ -43,6 +43,20 @@ "type": "boolean", "default": true }, + "apprise": { + "type": "boolean", + "default": false + }, + "apprisecmd": { + "type": "string" + }, + "ntfy": { + "type": "boolean", + "default": false + }, + "ntfycmd": { + "type": "string" + }, "runscrub": { "type": "boolean", "default": true diff --git a/usr/share/openmediavault/datamodels/rpc.snapraid.json b/usr/share/openmediavault/datamodels/rpc.snapraid.json index 5e7caba..c1b309c 100644 --- a/usr/share/openmediavault/datamodels/rpc.snapraid.json +++ b/usr/share/openmediavault/datamodels/rpc.snapraid.json @@ -59,6 +59,22 @@ "type": "boolean", "required": true }, + "apprise": { + "type": "boolean", + "required": true + }, + "apprisecmd": { + "type": "string", + "required": true + }, + "ntfy": { + "type": "boolean", + "required": true + }, + "ntfycmd": { + "type": "string", + "required": true + }, "runscrub": { "type": "boolean", "required": true diff --git a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml index 60ff777..9569714 100644 --- a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml +++ b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml @@ -60,6 +60,36 @@ data: name: sendmail label: _("Send mail") value: true + - type: checkbox + name: apprise + label: _("Use Apprise") + value: false + - type: textInput + name: apprisecmd + label: _("Apprise URL or config argument.") + hint: _("E.g. --config /myConfig.yml | ntfy://myTopic") + value: "" + language: shell + trim: true + validators: + pattern: + pattern: "^(--config\\s\\/[^\"'\\s][^\"'\\n]+|[\\w]+?:\\/\\/[^\"'\\s]+)$" + errorData: _("The field is invalid, it can't contain quotes and the path to the config file and URL can't contain spaces. Path to config file must be full (start with '/').") + - type: checkbox + name: ntfy + label: _("Use Ntfy") + value: false + - type: textInput + name: ntfycmd + label: _("Ntfy config command or host and/or topic.") + hint: _("E.g. ntfy.local/myTopic | myTopic | --config /someConfig.yml") + value: "OMV_SnapRAID" + trim: true + language: shell + validators: + pattern: + pattern: "^(([\\w\\.]+\\/)?[-\\w]{1,64}|--config\\s\\/[^\"'\\s]+)$" + errorData: _("The field can only contain a config command, a valid topic name or host and topic (don't add https:// at the start). No quotes or spaces are allowed. Path to config file must be full (start with '/').") - type: checkbox name: runscrub label: _("Run scrub") From 37986a18d4bd70e9880e5a961a679a07426c629f Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 01:15:24 -0500 Subject: [PATCH 02/15] support multiple URLs for apprise --- usr/sbin/omv-snapraid-diff | 6 ++---- .../omv-services-snapraid-settings-form-page.yaml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 1e98107..471c8e4 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -85,7 +85,6 @@ SNAPRAID_LOG="/var/log/snapraid.log" # 1 = Error happened # _send_notification() { - MAIL_FAIL=false APPRISE_FAIL=false NTFY_FAIL=false @@ -148,11 +147,10 @@ _send_email() { _send_apprise() { if _is_in_path apprise; then - if [ $3 != "" ]; then - apprise --title "$3 - $1" --body "$(cat $2)" $APPRISE_CMD + apprise -D --title "$3 - $1" --body "$(cat $2)" ${APPRISE_CMD//,/ } else - apprise --title "$1" --body "$(cat $2)" $APPRISE_CMD + apprise -D --title "$1" --body "$(cat $2)" ${APPRISE_CMD//,/ } fi if [ $? -eq 0 ]; then diff --git a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml index 9569714..66ebe1d 100644 --- a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml +++ b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml @@ -67,7 +67,7 @@ data: - type: textInput name: apprisecmd label: _("Apprise URL or config argument.") - hint: _("E.g. --config /myConfig.yml | ntfy://myTopic") + hint: _("E.g. --config /myConfig.yml | gotifys://myTopic\nYou can use multiple URLs, separate them with a ','.") value: "" language: shell trim: true From 96db1010b49306614fc31010965364be576c99f2 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 01:52:01 -0500 Subject: [PATCH 03/15] remove debug command for apprise --- usr/sbin/omv-snapraid-diff | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 471c8e4..7bb4925 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -148,9 +148,9 @@ _send_email() { _send_apprise() { if _is_in_path apprise; then if [ $3 != "" ]; then - apprise -D --title "$3 - $1" --body "$(cat $2)" ${APPRISE_CMD//,/ } + apprise --title "$3 - $1" --body "$(cat $2)" ${APPRISE_CMD//,/ } else - apprise -D --title "$1" --body "$(cat $2)" ${APPRISE_CMD//,/ } + apprise --title "$1" --body "$(cat $2)" ${APPRISE_CMD//,/ } fi if [ $? -eq 0 ]; then From 84775638b0bb74b542dcdf59ce08dd9bcbc76fdb Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 02:01:05 -0500 Subject: [PATCH 04/15] add debug log lines --- usr/sbin/omv-snapraid-diff | 2 ++ 1 file changed, 2 insertions(+) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 7bb4925..aa659fe 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -321,6 +321,8 @@ if $DEBUG; then _log "DEBUG: SCRUB_OLDER_THAN_DAYS: $SCRUB_OLDER_THAN_DAYS" _log "DEBUG: DEL_THRESHOLD: $DEL_THRESHOLD" _log "DEBUG: UPD_THRESHOLD: $UPD_THRESHOLD" + _log "DEBUG: APPRISE_CMD: $APPRISE_CMD" + _log "DEBUG: NTFY_CMD: $NTFY_CMD" fi SNAPRAID_BIN="/usr/bin/snapraid" From 35f39322363cdfd30131b857c3bc5f569e415c8f Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 07:13:01 -0500 Subject: [PATCH 05/15] rework scrub & move it to a function, better notification titles New method calculates the number of days it would take in an ideal scenario to scrub 100% of the data (100 * FREQUENCY / PERCENTAGE) e.g. for 10% every 7 days it would be 70 days. We take this number of days and assume that at least that number must be reached to consider the whole array scrubbed if it has been reached, the scrub starts from scratch again, rinse and repeat. If the user changes either the frequency or percentage or both, the scrub progress is reset back to 0. As it is right now, it does: run: 01 02 03 04 05 06 07 08 09 10 11 acc: 00 07 14 21 28 35 42 49 56 63 70->0 porc: 10 20 30 40 50 60 70 80 90 100 10 *acc = initial sum of days from first scrub to today (This reads, the first (01) run, the initially accumulated days will be 0 and the percentage that will end up scrubbed will be 10%) At day 11 the initially accumulated days will be equal to the max days, thus the scrub progress resets and all the data scrubs again by the percentage setup in OMV. Also, the notification titles have been improved: - If Scrub is not run, the title will only include Sync. - If Sync or Scrub fail, the title will be adjusted. --- usr/sbin/omv-snapraid-diff | 252 ++++++++++++++++++++++++++----------- 1 file changed, 176 insertions(+), 76 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index aa659fe..a12b3c4 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -3,7 +3,7 @@ # shellcheck disable=SC1090,SC1091,SC2086,SC2129,SC2162,SC2181,SC2219 ####################################################################### -# this is a helper script that keeps snapraid parity info in sync with +# this is a helper script that keeps SnapRAID parity info in sync with # your data. Here's how it works: # 1) it first calls diff to figure out if the parity info is out of sync # 2) if there are changed files (i.e. new, changed, moved or removed), @@ -64,7 +64,7 @@ fi # the following file is created by the OMV-GUI SR_DIFF_CONF="/etc/snapraid-diff.conf" -EMAIL_SUBJECT_PREFIX="SnapRAID" +NOTIFICATION_SUBJECT_PREFIX="SnapRAID" SNAPRAID_LOG="/var/log/snapraid.log" @@ -74,20 +74,20 @@ SNAPRAID_LOG="/var/log/snapraid.log" ################################################################ # -# name: _send_notification -# parameter : +# name: _send_notification +# parameter: # $1 = Subject # $2 = file with mailtext # Optional: # $3 = Who sent this message (full name), e.g. "Server Status-Report" # $4 = Answer to this address # return: 0 = All OK -# 1 = Error happened +# 1 = Error happened # _send_notification() { - MAIL_FAIL=false - APPRISE_FAIL=false - NTFY_FAIL=false + local MAIL_FAIL=false + local APPRISE_FAIL=false + local NTFY_FAIL=false if [ $SENDMAIL = "false" ] && [ $USE_APPRISE = "false" ] && [ $USE_NTFY = "false" ]; then _log "INFO: Notifications should be sent here - SENDMAIL, USE_APPRISE and USE_NTFY set to false" @@ -203,8 +203,8 @@ _is_in_path() { _log() { [[ "$*" =~ ^([A-Za-z]*):\ (.*) ]] && { - PRIORITY=${BASH_REMATCH[1]} # INFO, DEBUG, WARN - LOGMESSAGE=${BASH_REMATCH[2]} # the Log-Message + local PRIORITY=${BASH_REMATCH[1]} # INFO, DEBUG, WARN + local LOGMESSAGE=${BASH_REMATCH[2]} # the Log-Message } if $DEBUG; then @@ -229,10 +229,10 @@ _rmtmp() { ################################################################ # -# name : _check_success +# name : _check_success # parameter : none # return : 0 = last command was successful -# : 1 = last command ended with an error -> [ $? -gt 1 ] +# : 1 = last command ended with an error -> [ $? -gt 1 ] # _check_success() { if [ $? = 0 ]; then @@ -244,7 +244,7 @@ _check_success() { ################################################################ # -# name : _add_mail_header +# name : _add_mail_header # parameter : none # # adds mail text @@ -260,6 +260,73 @@ Warnings about 'Ignoring mount point' are possibly due to AUFS or mhddfs-Pooling EOF } +date_to_epoch() { + date -d "$1" +%s +} + +epoch_to_date() { + date -d @"$1" +%Y%m%d +} + +date_to_seconds() { + echo $(($1 * 24 * 60 * 60)) +} +_run_scrub() { + local CURRENT_DATE_EPOCH + local CURRENT_DATE_DATE + local DAYS_DIFF_DATE + local DAYS_DIFF_EPOCH + + SCRUB_SUCCESS="no" + + CURRENT_DATE_EPOCH=$(date +%s) + CURRENT_DATE_DATE=$(epoch_to_date $CURRENT_DATE_EPOCH) + + DAYS_DIFF_EPOCH=$((CURRENT_DATE_EPOCH - $(date_to_epoch $LAST_SCRUB_DATE))) + DAYS_DIFF_DATE=$(epoch_to_date $DAYS_DIFF_EPOCH) + + if [ $LAST_SCRUB_DATE -ne 0 ] && [ $DAYS_DIFF -lt $(($(date_to_seconds $SCRUB_FREQUENCY_IN_DAYS))) ]; then + _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run." + echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run. - $(date)" >>$TMP_OUTPUT + SCRUB_SUCCESS="none" + else + if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ne 0 ]; then + local MAX_NUM_DAYS=$((100 * SCRUB_FREQUENCY_IN_DAYS / SCRUB_PERCENT)) + if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ge $MAX_NUM_DAYS ]; then + # We reached the max num of days, this means we must restart and scrub everything again, + # in an ideal (non-file-changing) scenario we scrubbed all the data + ACCUMULATED_DAYS_SCRUBBED_FILES=0 + fi + fi + _log "INFO: SnapRAID SCRUB Job started" + _log "INFO: ----------------------------------------" + echo "SnapRAID SCRUB Job started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${ACCUMULATED_DAYS_SCRUBBED_FILES} >>$TMP_OUTPUT + + # wait for the job to finish + wait + if [ $? -eq 0 ]; then + SCRUB_SUCCESS="yes" + else + SCRUB_SUCCESS="no" + fi + + ((ACCUMULATED_DAYS_SCRUBBED_FILES += SCRUB_FREQUENCY_IN_DAYS)) + LAST_SCRUB_DATE=$CURRENT_DATE_DATE + + echo "$LAST_SCRUB_DATE" >$TMP_SCRUB_DIR/snapraid_scrub_last_date + echo "$LAST_SCRUB_FREQUENCY" >$TMP_SCRUB_DIR/snapraid_scrub_last_frequency + echo "$LAST_SCRUB_PERCENTAGE" >$TMP_SCRUB_DIR/snapraid_scrub_last_percentage + echo "$ACCUMULATED_DAYS_SCRUBBED_FILES" >$TMP_SCRUB_DIR/snapraid_scrub_accumulated_days + + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SCRUB Job finished" + echo "SnapRAID SCRUB Job finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + fi +} + ################################################################ ##### MAIN BODY of the script ################################################################ @@ -310,21 +377,6 @@ TMP_DIR="/tmp/snapraid" TMP_OUTPUT="$TMP_DIR/snapraid.out" TMP_SCRUB_DIR="/root/.snapraid" -if $DEBUG; then - _log "DEBUG: OMV_CONTENT_FILES: $OMV_CONTENT_FILES" - _log "DEBUG: PARITY_FILE: $PARITY_FILE" - _log "DEBUG: OMV_SNAPRAID_DISKS: $OMV_SNAPRAID_DISKS" - _log "DEBUG: TMP_OUTPUT: $TMP_OUTPUT" - _log "DEBUG: RUN_SCRUB: $RUN_SCRUB" - _log "DEBUG: SCRUB_FREQUENCY_IN_DAYS: $SCRUB_FREQUENCY_IN_DAYS" - _log "DEBUG: SCRUB_PERCENT: $SCRUB_PERCENT" - _log "DEBUG: SCRUB_OLDER_THAN_DAYS: $SCRUB_OLDER_THAN_DAYS" - _log "DEBUG: DEL_THRESHOLD: $DEL_THRESHOLD" - _log "DEBUG: UPD_THRESHOLD: $UPD_THRESHOLD" - _log "DEBUG: APPRISE_CMD: $APPRISE_CMD" - _log "DEBUG: NTFY_CMD: $NTFY_CMD" -fi - SNAPRAID_BIN="/usr/bin/snapraid" if [ -z $SNAPRAID_BIN ]; then _log "WARN: snapraid executable not found!" @@ -378,29 +430,79 @@ done ### Section to support SnapRAID Scrub ### mkdir -p $TMP_SCRUB_DIR -if [ -f $TMP_SCRUB_DIR/snapraid_scrub_counter ]; then - COUNTER=$(cat $TMP_SCRUB_DIR/snapraid_scrub_counter) -else - echo "1" >$TMP_SCRUB_DIR/snapraid_scrub_counter - COUNTER=1 -fi -# enable only one of the two below!!! -let COUNTER++ -# COUNTER=$(($COUNTER+0)) - # define Variables, if not set in snapraid-diff if [ -z ${SCRUB_PERCENT} ]; then SCRUB_PERCENT=100 _log "INFO: SCRUB_PERCENT not set in '$SR_DIFF_CONF'" _log "INFO: set it to $SCRUB_PERCENT" fi -if [ -z ${SCRUB_OLDER_THAN_DAYS} ]; then - SCRUB_OLDER_THAN_DAYS=0 - _log "INFO: SCRUB_OLDER_THAN_DAYS not set in '$SR_DIFF_CONF'" - _log "INFO: set it to $SCRUB_OLDER_THAN_DAYS" +if [ -z ${SCRUB_FREQUENCY_IN_DAYS} ]; then + SCRUB_FREQUENCY_IN_DAYS=0 + _log "INFO: SCRUB_FREQUENCY_IN_DAYS not set in '$SR_DIFF_CONF'" + _log "INFO: set it to $SCRUB_FREQUENCY_IN_DAYS" +fi + +# for scrub +RESET_SCRUB_CALC="false" + +if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_date ]; then + LAST_SCRUB_DATE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_date) +else + LAST_SCRUB_DATE=0 +fi + +if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_freq ]; then + LAST_SCRUB_FREQUENCY=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_frequency) + if $LAST_SCRUB_FREQUENCY != $SCRUB_FREQUENCY_IN_DAYS; then + RESET_SCRUB_CALC="true" + fi +fi + +LAST_SCRUB_FREQUENCY=$SCRUB_FREQUENCY_IN_DAYS + +if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_percentage ]; then + LAST_SCRUB_PERCENTAGE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_percentage) + if $LAST_SCRUB_PERCENTAGE != $SCRUB_PERCENT; then + RESET_SCRUB_CALC="true" + fi +fi + +LAST_SCRUB_PERCENTAGE=$SCRUB_PERCENT + +# We'll store the accumulated days of the scrubbed files, +# that way if we use 10% every 7 days, we are sure in 70 days the whole array is scrubbed +if [ -f $TMP_SCRUB_DIR/snapraid_scrub_accumulated_days ]; then + ACCUMULATED_DAYS_SCRUBBED_FILES=$(cat $TMP_SCRUB_DIR/snapraid_scrub_accumulated_days) +else + ACCUMULATED_DAYS_SCRUBBED_FILES=0 +fi + +if [ $RESET_SCRUB_CALC = "true" ]; then + # If the user changes either the scrub frequency or percentage then we start the scrub from 0, + # resetting the current progress. + ACCUMULATED_DAYS_SCRUBBED_FILES=0 + LAST_SCRUB_DATE=0 fi ### End Scrub Section ### +if $DEBUG; then + _log "DEBUG: OMV_CONTENT_FILES: $OMV_CONTENT_FILES" + _log "DEBUG: PARITY_FILE: $PARITY_FILE" + _log "DEBUG: OMV_SNAPRAID_DISKS: $OMV_SNAPRAID_DISKS" + _log "DEBUG: TMP_OUTPUT: $TMP_OUTPUT" + _log "DEBUG: RUN_SCRUB: $RUN_SCRUB" + _log "DEBUG: SCRUB_FREQUENCY_IN_DAYS: $SCRUB_FREQUENCY_IN_DAYS" + _log "DEBUG: SCRUB_PERCENT: $SCRUB_PERCENT" + _log "DEBUG: DEL_THRESHOLD: $DEL_THRESHOLD" + _log "DEBUG: UPD_THRESHOLD: $UPD_THRESHOLD" + _log "DEBUG: APPRISE_CMD: $APPRISE_CMD" + _log "DEBUG: NTFY_CMD: $NTFY_CMD" + _log "DEBUG: LAST_SCRUB_DATE: $LAST_SCRUB_DATE" + _log "DEBUG: LAST_SCRUB_FREQUENCY: $LAST_SCRUB_FREQUENCY" + _log "DEBUG: LAST_SCRUB_PERCENTAGE: $LAST_SCRUB_PERCENTAGE" + _log "DEBUG: ACCUMULATED_DAYS_SCRUBBED_FILES: $ACCUMULATED_DAYS_SCRUBBED_FILES" +fi + ################################################################ ## Start of the script _log "INFO: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" @@ -434,7 +536,7 @@ for OMV_DISK in $OMV_SNAPRAID_DISKS; do echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >>$TMP_OUTPUT echo "Lastly, use 'snapraid sync' to update the parity files." >>$TMP_OUTPUT - _send_notification "$EMAIL_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" _check_success && _rmtmp _log "WARN: script exits here due to missing disk ..." exit 1 @@ -442,7 +544,7 @@ for OMV_DISK in $OMV_SNAPRAID_DISKS; do done ################################################################ -# start snapraid DIFF command +# start SnapRAID DIFF command _log "INFO: SnapRAID DIFF started" echo "SnapRAID DIFF started - $(date)" >>$TMP_OUTPUT @@ -530,7 +632,7 @@ if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $C # Proceed with sync if no check errors if [ $UPD_DEL_CHECK -eq 0 ]; then - # start snapraid SYNC + # start SnapRAID SYNC _log "INFO: " _log "INFO: SnapRAID SYNC Job started" _log "INFO: ----------------------------------------" @@ -541,60 +643,58 @@ if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $C # we should eventually grep % if [ ${PREHASH_VALUE} -eq 1 ]; then /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} --pre-hash >>$TMP_OUTPUT + + #wait for the job to finish + wait + SYNC_EXIT_CODE=$? else /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} >>$TMP_OUTPUT + + #wait for the job to finish + wait + SYNC_EXIT_CODE=$? fi - #wait for the job to finish - wait _log "INFO: ----------------------------------------" _log "INFO: SnapRAID SYNC Job finished" echo "SnapRAID SYNC Job finished - $(date)" >>$TMP_OUTPUT echo "----------------------------------------" >>$TMP_OUTPUT - # start snapraid SCRUB - if [ $RUN_SCRUB = "true" ]; then - if [ $COUNTER -lt $SCRUB_FREQUENCY_IN_DAYS ]; then - _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($COUNTER). No scrub was run." - echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($COUNTER). No scrub was run. - $(date)" >>$TMP_OUTPUT - echo "$COUNTER" >$TMP_SCRUB_DIR/snapraid_scrub_counter + # start SnapRAID SCRUB + if [ $SYNC_EXIT_CODE -eq 0 ]; then + if [ $RUN_SCRUB = "true" ]; then + _run_scrub + if [ $SCRUB_SUCCESS = "yes" ]; then + NOTIFICATION_TITLE="Sync & Scrub Job COMPLETED" + elif [ $SCRUB_SUCCESS = "no" ]; then + NOTIFICATION_TITLE="Sync job COMPLETED & Scrub Job FAILED" + else + # No scrub was executed because of the scrub conditions + NOTIFICATION_TITLE="Sync Job COMPLETED" + fi else - _log "INFO: SnapRAID SCRUB Job started" - _log "INFO: ----------------------------------------" - echo "SnapRAID SCRUB Job started - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT - /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${SCRUB_OLDER_THAN_DAYS} >>$TMP_OUTPUT - - # wait for the job to finish - wait - - # Reset counter - echo "1" >$TMP_SCRUB_DIR/snapraid_scrub_counter - - _log "INFO: ----------------------------------------" - _log "INFO: SnapRAID SCRUB Job finished" - echo "SnapRAID SCRUB Job finished - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT + echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT + _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." + NOTIFICATION_TITLE="Sync Job COMPLETED" fi else - echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT - _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." + NOTIFICATION_TITLE="Sync Job FAILED" fi - # Send success email - _send_notification "$EMAIL_SUBJECT_PREFIX - Sync/Scrub Job COMPLETED" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + # Send job finished email + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - $NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" # If update threshold exceeded elif [ $UPD_DEL_CHECK -eq 1 ]; then - _send_notification "$EMAIL_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" # If delete threshold exceeded elif [ $UPD_DEL_CHECK -eq 2 ]; then - _send_notification "$EMAIL_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) else - _send_notification "$EMAIL_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${EMAIL_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" fi _check_success && _rmtmp else From 2613c16c4f9b5865a0877566dfc8392f93dd0c97 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 11:41:39 -0500 Subject: [PATCH 06/15] better command split for apprise, added notification logging --- .../snapraid/files/etc-snapraid-diff_conf.j2 | 3 +- usr/sbin/omv-snapraid-diff | 74 +++++++++++++++---- ...-services-snapraid-settings-form-page.yaml | 6 +- 3 files changed, 64 insertions(+), 19 deletions(-) diff --git a/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 b/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 index 6b2d6ed..5e9a2ed 100644 --- a/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 +++ b/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 @@ -8,6 +8,5 @@ DEL_THRESHOLD={{ config.delthreshold }} SCRUB_PERCENT={{ config.scrubpercent }} {{ 'USE_APPRISE=true' if config.apprise | to_bool else 'USE_APPRISE=false' }} {{ 'USE_NTFY=true' if config.ntfy | to_bool else 'USE_NTFY=false' }} -APPRISE_CMD={{ config.apprisecmd }} +APPRISE_CMD='{{ config.apprisecmd }}' NTFY_CMD={{ config.ntfycmd }} - diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index a12b3c4..cd6a962 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -67,6 +67,11 @@ SR_DIFF_CONF="/etc/snapraid-diff.conf" NOTIFICATION_SUBJECT_PREFIX="SnapRAID" SNAPRAID_LOG="/var/log/snapraid.log" +SNAPRAID_NOTIF_LOG="/var/log/snapraid-notif.log" + +if [ -f "${SNAPRAID_NOTIF_LOG}" ]; then + rm -f "$SNAPRAID_NOTIF_LOG" +fi ################################################################ ##### Functions @@ -137,27 +142,45 @@ _send_email() { mail -c "$OMV_MAIL_secondarymail" -s "$1" "$OMV_MAIL_primarymail" <$2 if [ $? -eq 0 ]; then - _log "INFO: Full statusreport sent to '$MAIL'" + _log "INFO: Full status report sent to '$MAIL'" return 0 else - _log "WARN: Mail with full statusreport could not be sent. There was an error. Please have a look at /var/log/mail.log" + _log "WARN: Mail with full status report could not be sent. There was an error. Please have a look at /var/log/mail.log" return 1 fi } _send_apprise() { + local CMD + local APPRISE_CMD_ARRAY + local VERBOSE_LEVEL + + if $DEBUG; then + VERBOSE_LEVEL=-D + else + VERBOSE_LEVEL=-vv + fi + + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG + echo "APPRISE LOGS - $(date +'%Y/%m/%d %H:%M:%S')" >>$SNAPRAID_NOTIF_LOG + if _is_in_path apprise; then - if [ $3 != "" ]; then - apprise --title "$3 - $1" --body "$(cat $2)" ${APPRISE_CMD//,/ } - else - apprise --title "$1" --body "$(cat $2)" ${APPRISE_CMD//,/ } - fi + read -r -a APPRISE_CMD_ARRAY <<<"$(_split_cmd $APPRISE_CMD)" + for CMD in "${APPRISE_CMD_ARRAY[@]}"; do + if [ $3 != "" ]; then + apprise $VERBOSE_LEVEL --title "$3 - $1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG + else + apprise $VERBOSE_LEVEL --title "$1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG + fi + done + wait if [ $? -eq 0 ]; then - _log "INFO: Full statusreport sent through 'Apprise'" + _log "INFO: Full status report sent through 'Apprise'" return 0 else - _log "WARN: Apprise notification with full statusreport could not be sent. There was an error." + _log "WARN: Apprise notification with full status report could not be sent. Please have a look at $SNAPRAID_NOTIF_LOG." return 1 fi else @@ -168,17 +191,29 @@ _send_apprise() { _send_ntfy() { if _is_in_path ntfy; then + local VERBOSE_LEVEL + + if $DEBUG; then + VERBOSE_LEVEL=debug + else + VERBOSE_LEVEL=warn + fi + + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG + echo "NTFY LOGS - $(date +'%Y/%m/%d %H:%M:%S')" >>$SNAPRAID_NOTIF_LOG + if [ $3 != "" ]; then - ntfy publish --title "$3 - $1" --message "$(cat $2)" $NTFY_CMD + ntfy publish --log-level $VERBOSE_LEVEL --title "$3 - $1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG else - ntfy publish --title "$1" --message "$(cat $2)" $NTFY_CMD + ntfy publish --log-level $VERBOSE_LEVEL --title "$1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG fi if [ $? -eq 0 ]; then - _log "INFO: Full statusreport sent through 'Ntfy'" + _log "INFO: Full status report sent through 'Ntfy'" return 0 else - _log "WARN: Ntfy notification with full statusreport could not be sent. There was an error." + _log "WARN: Ntfy notification with full status report could not be sent. Please have a look at $SNAPRAID_NOTIF_LOG." return 1 fi else @@ -214,7 +249,18 @@ _log() { echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG fi } +_split_cmd() { + local CMD_QUOTED="$1" + + local IFS + local CMDS + IFS=';' + read -r -a CMDS <<<"${CMD_QUOTED}" + + IFS=" " + echo "${CMDS[*]}" +} ################################################################ # # name : _rmtmp @@ -495,7 +541,7 @@ if $DEBUG; then _log "DEBUG: SCRUB_PERCENT: $SCRUB_PERCENT" _log "DEBUG: DEL_THRESHOLD: $DEL_THRESHOLD" _log "DEBUG: UPD_THRESHOLD: $UPD_THRESHOLD" - _log "DEBUG: APPRISE_CMD: $APPRISE_CMD" + _log "DEBUG: APPRISE_CMD: ${APPRISE_CMD}" _log "DEBUG: NTFY_CMD: $NTFY_CMD" _log "DEBUG: LAST_SCRUB_DATE: $LAST_SCRUB_DATE" _log "DEBUG: LAST_SCRUB_FREQUENCY: $LAST_SCRUB_FREQUENCY" diff --git a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml index 66ebe1d..8d4fb04 100644 --- a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml +++ b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml @@ -67,13 +67,13 @@ data: - type: textInput name: apprisecmd label: _("Apprise URL or config argument.") - hint: _("E.g. --config /myConfig.yml | gotifys://myTopic\nYou can use multiple URLs, separate them with a ','.") + hint: _("E.g. --config /myConfig.yml | gotifys://myHost/myToken | mailtos://userid:pass@domain.com;gotifys://myHost/myToken
You can use multiple URLs, separate them with a ';'.") value: "" language: shell trim: true validators: pattern: - pattern: "^(--config\\s\\/[^\"'\\s][^\"'\\n]+|[\\w]+?:\\/\\/[^\"'\\s]+)$" + pattern: "^(--config\\s\\/[^\"'\\s\\|\\&][^\"'\\n]+|[\\w]+?:\\/\\/[^\"'\\s\\|]+)$" errorData: _("The field is invalid, it can't contain quotes and the path to the config file and URL can't contain spaces. Path to config file must be full (start with '/').") - type: checkbox name: ntfy @@ -88,7 +88,7 @@ data: language: shell validators: pattern: - pattern: "^(([\\w\\.]+\\/)?[-\\w]{1,64}|--config\\s\\/[^\"'\\s]+)$" + pattern: "^(([\\w\\.]+\\/)?[-\\w]{1,64}|--config\\s\\/[^\"'\\s\\|\\&]+)$" errorData: _("The field can only contain a config command, a valid topic name or host and topic (don't add https:// at the start). No quotes or spaces are allowed. Path to config file must be full (start with '/').") - type: checkbox name: runscrub From dbf568a99b6962941b0d9068d46718939d19df30 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sat, 1 Jun 2024 22:59:20 -0500 Subject: [PATCH 07/15] move things to functions, style changes --- usr/sbin/omv-snapraid-diff | 536 ++++++++++++++++++++----------------- 1 file changed, 295 insertions(+), 241 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index cd6a962..4f4fe54 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -89,6 +89,7 @@ fi # return: 0 = All OK # 1 = Error happened # + _send_notification() { local MAIL_FAIL=false local APPRISE_FAIL=false @@ -154,6 +155,7 @@ _send_apprise() { local CMD local APPRISE_CMD_ARRAY local VERBOSE_LEVEL + local APPRISE_RETURN_CODE if $DEBUG; then VERBOSE_LEVEL=-D @@ -161,22 +163,25 @@ _send_apprise() { VERBOSE_LEVEL=-vv fi - echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG echo "APPRISE LOGS - $(date +'%Y/%m/%d %H:%M:%S')" >>$SNAPRAID_NOTIF_LOG + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG if _is_in_path apprise; then read -r -a APPRISE_CMD_ARRAY <<<"$(_split_cmd $APPRISE_CMD)" for CMD in "${APPRISE_CMD_ARRAY[@]}"; do if [ $3 != "" ]; then - apprise $VERBOSE_LEVEL --title "$3 - $1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG + apprise $VERBOSE_LEVEL --title "$3 - $1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG 2>&1 + wait + APPRISE_RETURN_CODE=$? else - apprise $VERBOSE_LEVEL --title "$1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG + apprise $VERBOSE_LEVEL --title "$1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG 2>&1 + wait + APPRISE_RETURN_CODE=$? fi done - wait - if [ $? -eq 0 ]; then + if [ $APPRISE_RETURN_CODE -eq 0 ]; then _log "INFO: Full status report sent through 'Apprise'" return 0 else @@ -190,26 +195,31 @@ _send_apprise() { } _send_ntfy() { - if _is_in_path ntfy; then - local VERBOSE_LEVEL + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG + echo "NTFY LOGS - $(date +'%Y/%m/%d %H:%M:%S')" >>$SNAPRAID_NOTIF_LOG + echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG - if $DEBUG; then - VERBOSE_LEVEL=debug - else - VERBOSE_LEVEL=warn - fi + local VERBOSE_LEVEL + local NTFY_RETURN_CODE - echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG - echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG - echo "NTFY LOGS - $(date +'%Y/%m/%d %H:%M:%S')" >>$SNAPRAID_NOTIF_LOG + if $DEBUG; then + VERBOSE_LEVEL=debug + else + VERBOSE_LEVEL=warn + fi + if _is_in_path ntfy; then if [ $3 != "" ]; then - ntfy publish --log-level $VERBOSE_LEVEL --title "$3 - $1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG + ntfy publish --log-level $VERBOSE_LEVEL --title "$3 - $1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1 + wait + NTFY_RETURN_CODE=$? else - ntfy publish --log-level $VERBOSE_LEVEL --title "$1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG + ntfy publish --log-level $VERBOSE_LEVEL --title "$1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1 + wait + NTFY_RETURN_CODE=$? fi - if [ $? -eq 0 ]; then + if [ $NTFY_RETURN_CODE -eq 0 ]; then _log "INFO: Full status report sent through 'Ntfy'" return 0 else @@ -229,12 +239,14 @@ _is_in_path() { return 1 fi } + ################################################################ # # name : _log # parameter : $LOGMESSAGE : logmessage in format "PRIORITY: MESSAGE" # return : none # + _log() { [[ "$*" =~ ^([A-Za-z]*):\ (.*) ]] && { @@ -249,6 +261,7 @@ _log() { echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG fi } + _split_cmd() { local CMD_QUOTED="$1" @@ -261,6 +274,7 @@ _split_cmd() { IFS=" " echo "${CMDS[*]}" } + ################################################################ # # name : _rmtmp @@ -276,12 +290,12 @@ _rmtmp() { ################################################################ # # name : _check_success -# parameter : none +# parameter : return code to check # return : 0 = last command was successful # : 1 = last command ended with an error -> [ $? -gt 1 ] # _check_success() { - if [ $? = 0 ]; then + if [ $1 = 0 ]; then return 0 else return 1 @@ -317,60 +331,9 @@ epoch_to_date() { date_to_seconds() { echo $(($1 * 24 * 60 * 60)) } -_run_scrub() { - local CURRENT_DATE_EPOCH - local CURRENT_DATE_DATE - local DAYS_DIFF_DATE - local DAYS_DIFF_EPOCH - - SCRUB_SUCCESS="no" - - CURRENT_DATE_EPOCH=$(date +%s) - CURRENT_DATE_DATE=$(epoch_to_date $CURRENT_DATE_EPOCH) - - DAYS_DIFF_EPOCH=$((CURRENT_DATE_EPOCH - $(date_to_epoch $LAST_SCRUB_DATE))) - DAYS_DIFF_DATE=$(epoch_to_date $DAYS_DIFF_EPOCH) - - if [ $LAST_SCRUB_DATE -ne 0 ] && [ $DAYS_DIFF -lt $(($(date_to_seconds $SCRUB_FREQUENCY_IN_DAYS))) ]; then - _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run." - echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run. - $(date)" >>$TMP_OUTPUT - SCRUB_SUCCESS="none" - else - if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ne 0 ]; then - local MAX_NUM_DAYS=$((100 * SCRUB_FREQUENCY_IN_DAYS / SCRUB_PERCENT)) - if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ge $MAX_NUM_DAYS ]; then - # We reached the max num of days, this means we must restart and scrub everything again, - # in an ideal (non-file-changing) scenario we scrubbed all the data - ACCUMULATED_DAYS_SCRUBBED_FILES=0 - fi - fi - _log "INFO: SnapRAID SCRUB Job started" - _log "INFO: ----------------------------------------" - echo "SnapRAID SCRUB Job started - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT - /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${ACCUMULATED_DAYS_SCRUBBED_FILES} >>$TMP_OUTPUT - - # wait for the job to finish - wait - if [ $? -eq 0 ]; then - SCRUB_SUCCESS="yes" - else - SCRUB_SUCCESS="no" - fi - - ((ACCUMULATED_DAYS_SCRUBBED_FILES += SCRUB_FREQUENCY_IN_DAYS)) - LAST_SCRUB_DATE=$CURRENT_DATE_DATE - - echo "$LAST_SCRUB_DATE" >$TMP_SCRUB_DIR/snapraid_scrub_last_date - echo "$LAST_SCRUB_FREQUENCY" >$TMP_SCRUB_DIR/snapraid_scrub_last_frequency - echo "$LAST_SCRUB_PERCENTAGE" >$TMP_SCRUB_DIR/snapraid_scrub_last_percentage - echo "$ACCUMULATED_DAYS_SCRUBBED_FILES" >$TMP_SCRUB_DIR/snapraid_scrub_accumulated_days - _log "INFO: ----------------------------------------" - _log "INFO: SnapRAID SCRUB Job finished" - echo "SnapRAID SCRUB Job finished - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT - fi +seconds_to_days() { + echo $(($1 / 60 / 60 / 24)) } ################################################################ @@ -549,205 +512,296 @@ if $DEBUG; then _log "DEBUG: ACCUMULATED_DAYS_SCRUBBED_FILES: $ACCUMULATED_DAYS_SCRUBBED_FILES" fi -################################################################ -## Start of the script -_log "INFO: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -_log "INFO: SnapRAID Job started - Script-version: $SCRIPTVERSION" -_log "INFO: SnapRAID-Version: $SNAPRAIDVERSION" -_add_mail_header # all info above is in that function - -_log "INFO: Checking 'disks' in '$SNAPRAID_CONF'" -### check if all hdds in the config are there -for OMV_DISK in $OMV_SNAPRAID_DISKS; do - - if $DEBUG; then _log "DEBUG: found: OMV_DISK: $OMV_DISK"; fi - - if ! grep $OMV_DISK /etc/mtab >/dev/null 2>&1; then - - #if [ ! -b $OMV_DISK ]; then # doesn't work - no block device found :o - _log "WARN: ERROR: Disk $OMV_DISK was not found or is not mounted!" - _log "WARN: Sync job cancelled to prevent data loss." - _log "WARN: Please check your disks!" - _log "WARN: Replace damaged Disk and proceed with 'snapraid fix'" - _log "WARN: to rebuild the raid array and recover your data." - _log "WARN: Then, use 'snapraid check' to verify the integrity of the recovered files." - _log "WARN: Lastly, use 'snapraid sync' to update the parity files." - - echo "ERROR - Data Disk $OMV_DISK not found or is not mounted!" >>$TMP_OUTPUT - echo "Sync job cancelled to prevent data loss." >>$TMP_OUTPUT - echo " " >>$TMP_OUTPUT - echo "Please check your disks!" >>$TMP_OUTPUT - echo "Replace damaged Disk and proceed with 'snapraid fix'" >>$TMP_OUTPUT - echo "to rebuild the raid array and recover your data." >>$TMP_OUTPUT - echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >>$TMP_OUTPUT - echo "Lastly, use 'snapraid sync' to update the parity files." >>$TMP_OUTPUT - - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" - _check_success && _rmtmp - _log "WARN: script exits here due to missing disk ..." - exit 1 - fi -done +_start_job() { + ################################################################ + ## Start of the script + _log "INFO: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + _log "INFO: SnapRAID Job started - Script-version: $SCRIPTVERSION" + _log "INFO: SnapRAID-Version: $SNAPRAIDVERSION" + _add_mail_header # all info above is in that function -################################################################ -# start SnapRAID DIFF command -_log "INFO: SnapRAID DIFF started" -echo "SnapRAID DIFF started - $(date)" >>$TMP_OUTPUT - -# run diff: -# output: stdout: WARNINGS only (filtering 'Ignoring mount point') and $TMP_OUTPUT (all WARNINGS) -/usr/bin/snapraid diff --conf ${SNAPRAID_CONF} 2>&1 | tee -a $TMP_OUTPUT | grep WARNING | grep -v 'WARNING! Ignoring mount point' | while read line; do - _log "WARN: $line" -done + _check_disks -# wait for the above cmd to finish -wait + ################################################################ + # start SnapRAID DIFF command -_log "INFO: SnapRAID DIFF finished" -_log "INFO: ----------------------------------------" -echo "SnapRAID DIFF finished - $(date)" >>$TMP_OUTPUT -echo "----------------------------------------" >>$TMP_OUTPUT + _start_diff +} -# read the values of deleted, added, etc. files since the last sync -DEL_COUNT=$(grep -w '^ \{1,\}[0-9]* removed$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) -ADD_COUNT=$(grep -w '^ \{1,\}[0-9]* added$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) -MOVE_COUNT=$(grep -w '^ \{1,\}[0-9]* moved$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) -COPY_COUNT=$(grep -w '^ \{1,\}[0-9]* copied$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) -UPDATE_COUNT=$(grep -w '^ \{1,\}[0-9]* updated$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) +_check_disks() { + _log "INFO: Checking 'disks' in '$SNAPRAID_CONF'" -_log "INFO: SUMMARY of changes since last sync:" -_log "INFO: Added: [$ADD_COUNT] - Deleted: [$DEL_COUNT] - Moved: [$MOVE_COUNT] - Copied: [$COPY_COUNT] - Updated: [$UPDATE_COUNT]" + local OMV_DISK + local NOTIFICATION_RETURN_CODE -# set pre-hash option -PREHASH_VALUE=$(omv_config_get "/config/services/snapraid/prehash") -if $DEBUG; then - _log "DEBUG: Pre-Hash option: ${PREHASH_VALUE}" -fi + ### check if all hdds in the config are there -# check if files have changed -if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $COPY_COUNT -gt 0 ] || [ $UPDATE_COUNT -gt 0 ]; then + for OMV_DISK in $OMV_SNAPRAID_DISKS; do + if $DEBUG; then + _log "DEBUG: found: OMV_DISK: $OMV_DISK" + fi - UPD_THRESHOLD=$(omv_config_get "/config/services/snapraid/updthreshold") - DEL_THRESHOLD=$(omv_config_get "/config/services/snapraid/delthreshold") + if ! grep $OMV_DISK /etc/mtab >/dev/null 2>&1; then + #if [ ! -b $OMV_DISK ]; then # doesn't work - no block device found :o + _log "WARN: ERROR: Disk $OMV_DISK was not found or is not mounted!" + _log "WARN: Sync job cancelled to prevent data loss." + _log "WARN: Please check your disks!" + _log "WARN: Replace damaged Disk and proceed with 'snapraid fix'" + _log "WARN: to rebuild the raid array and recover your data." + _log "WARN: Then, use 'snapraid check' to verify the integrity of the recovered files." + _log "WARN: Lastly, use 'snapraid sync' to update the parity files." + + echo "ERROR - Data Disk $OMV_DISK not found or is not mounted!" >>$TMP_OUTPUT + echo "Sync job cancelled to prevent data loss." >>$TMP_OUTPUT + echo " " >>$TMP_OUTPUT + echo "Please check your disks!" >>$TMP_OUTPUT + echo "Replace damaged Disk and proceed with 'snapraid fix'" >>$TMP_OUTPUT + echo "to rebuild the raid array and recover your data." >>$TMP_OUTPUT + echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >>$TMP_OUTPUT + echo "Lastly, use 'snapraid sync' to update the parity files." >>$TMP_OUTPUT + + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + NOTIFICATION_RETURN_CODE=$? + _check_success $NOTIFICATION_RETURN_CODE && _rmtmp + _log "WARN: script exits here due to missing disk ..." + exit 1 + fi + done +} - # New Logic for v 1.2.0 - check whether update threshold is enabled or not - if [ $UPD_THRESHOLD -gt 0 ]; then - # Update threshold is enabled; check if count exceeds threshold - if [ $UPDATE_COUNT -gt $UPD_THRESHOLD ]; then - # Too many files updated; inform user and don't proceed with sync - _log "INFO: Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." - _log "INFO: Please run sync manually if this is not an error condition." - echo "Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." >>$TMP_OUTPUT - echo "Please run sync manually if this is not an error condition" >>$TMP_OUTPUT +_start_sync() { + _log "INFO: " + _log "INFO: SnapRAID SYNC Job started" + _log "INFO: ----------------------------------------" + echo " " >>$TMP_OUTPUT + echo "SnapRAID SYNC Job started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT - # Increment check variable - UPD_DEL_CHECK=$((UPD_DEL_CHECK + 1)) - else - # Update threshold is set but not exceeded. Inform user that UPDATE_COUNT is below threshold and allow to proceed - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." >>$TMP_OUTPUT - fi + local SYNC_EXIT_CODE + local NOTIFICATION_TITLE + + # we should eventually grep % + if [ ${PREHASH_VALUE} -eq 1 ]; then + /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} --pre-hash >>$TMP_OUTPUT + + #wait for the job to finish + wait + SYNC_EXIT_CODE=$? else - # UPD_THRESHOLD is disabled, so we reflect this information in email and log - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." >>$TMP_OUTPUT + /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} >>$TMP_OUTPUT + + #wait for the job to finish + wait + SYNC_EXIT_CODE=$? fi - # New Logic for v 1.2.0 - check whether delete threshold is enabled or not - if [ $DEL_THRESHOLD -gt 0 ]; then - # Delete threshold is enabled; check if count exceeds threshold - if [ $DEL_COUNT -gt $DEL_THRESHOLD ]; then - # Too many files deleted; inform user and don't proceed with sync - _log "INFO: Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." - _log "INFO: Please run sync manually if this is not an error condition." - echo "Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." >>$TMP_OUTPUT - echo "Please run sync manually if this is not an error condition" >>$TMP_OUTPUT - - # Increment check variable - UPD_DEL_CHECK=$((UPD_DEL_CHECK + 2)) + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SYNC Job finished" + echo "SnapRAID SYNC Job finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + + # start SnapRAID SCRUB + if [ $SYNC_EXIT_CODE -eq 0 ]; then + if [ $RUN_SCRUB = "true" ]; then + _start_scrub + if [ $SCRUB_SUCCESS = "yes" ]; then + NOTIFICATION_TITLE="Sync & Scrub Job COMPLETED" + elif [ $SCRUB_SUCCESS = "no" ]; then + NOTIFICATION_TITLE="Sync job COMPLETED & Scrub Job FAILED" + else + # No scrub was executed because of the scrub conditions + NOTIFICATION_TITLE="Sync Job COMPLETED" + fi else - # Delete threshold is set but not exceeded. Inform user that DEL_COUNT is below threshold and allow to proceed - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." >>$TMP_OUTPUT + echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT + _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." + NOTIFICATION_TITLE="Sync Job COMPLETED" fi else - # DEL_THRESHOLD is disabled, so we reflect this information in email and log - _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." - echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." >>$TMP_OUTPUT + NOTIFICATION_TITLE="Sync Job FAILED" fi - # Proceed with sync if no check errors - if [ $UPD_DEL_CHECK -eq 0 ]; then - # start SnapRAID SYNC - _log "INFO: " - _log "INFO: SnapRAID SYNC Job started" - _log "INFO: ----------------------------------------" - echo "" >>$TMP_OUTPUT - echo "SnapRAID SYNC Job started - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT + # Send job finished email + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - $NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" +} - # we should eventually grep % - if [ ${PREHASH_VALUE} -eq 1 ]; then - /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} --pre-hash >>$TMP_OUTPUT +_start_diff() { + _log "INFO: SnapRAID DIFF started" + echo "SnapRAID DIFF started - $(date)" >>$TMP_OUTPUT + + local UPD_THRESHOLD + local DEL_THRESHOLD + local NOTIFICATION_RETURN_CODE + local line + + # run diff: + # output: stdout: WARNINGS only (filtering 'Ignoring mount point') and $TMP_OUTPUT (all WARNINGS) + /usr/bin/snapraid diff --conf ${SNAPRAID_CONF} 2>&1 | tee -a $TMP_OUTPUT | grep WARNING | grep -v 'WARNING! Ignoring mount point' | while read line; do + _log "WARN: $line" + done + + # wait for the above cmd to finish + wait + + _log "INFO: SnapRAID DIFF finished" + _log "INFO: ----------------------------------------" + echo "SnapRAID DIFF finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + + # read the values of deleted, added, etc. files since the last sync + DEL_COUNT=$(grep -w '^ \{1,\}[0-9]* removed$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) + ADD_COUNT=$(grep -w '^ \{1,\}[0-9]* added$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) + MOVE_COUNT=$(grep -w '^ \{1,\}[0-9]* moved$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) + COPY_COUNT=$(grep -w '^ \{1,\}[0-9]* copied$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) + UPDATE_COUNT=$(grep -w '^ \{1,\}[0-9]* updated$' $TMP_OUTPUT | sed 's/^ *//g' | cut -d ' ' -f1) + + _log "INFO: SUMMARY of changes since last sync:" + _log "INFO: Added: [$ADD_COUNT] - Deleted: [$DEL_COUNT] - Moved: [$MOVE_COUNT] - Copied: [$COPY_COUNT] - Updated: [$UPDATE_COUNT]" + + # set pre-hash option + PREHASH_VALUE=$(omv_config_get "/config/services/snapraid/prehash") + if $DEBUG; then + _log "DEBUG: Pre-Hash option: ${PREHASH_VALUE}" + fi - #wait for the job to finish - wait - SYNC_EXIT_CODE=$? - else - /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} >>$TMP_OUTPUT + # check if files have changed + if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $COPY_COUNT -gt 0 ] || [ $UPDATE_COUNT -gt 0 ]; then - #wait for the job to finish - wait - SYNC_EXIT_CODE=$? - fi + UPD_THRESHOLD=$(omv_config_get "/config/services/snapraid/updthreshold") + DEL_THRESHOLD=$(omv_config_get "/config/services/snapraid/delthreshold") - _log "INFO: ----------------------------------------" - _log "INFO: SnapRAID SYNC Job finished" - echo "SnapRAID SYNC Job finished - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT + # New Logic for v 1.2.0 - check whether update threshold is enabled or not + if [ $UPD_THRESHOLD -gt 0 ]; then + # Update threshold is enabled; check if count exceeds threshold + if [ $UPDATE_COUNT -gt $UPD_THRESHOLD ]; then + # Too many files updated; inform user and don't proceed with sync + _log "INFO: Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." + _log "INFO: Please run sync manually if this is not an error condition." + echo "Number of updated files ($UPDATE_COUNT) exceeded threshold ($UPD_THRESHOLD). NOT proceeding with sync job." >>$TMP_OUTPUT + echo "Please run sync manually if this is not an error condition" >>$TMP_OUTPUT - # start SnapRAID SCRUB - if [ $SYNC_EXIT_CODE -eq 0 ]; then - if [ $RUN_SCRUB = "true" ]; then - _run_scrub - if [ $SCRUB_SUCCESS = "yes" ]; then - NOTIFICATION_TITLE="Sync & Scrub Job COMPLETED" - elif [ $SCRUB_SUCCESS = "no" ]; then - NOTIFICATION_TITLE="Sync job COMPLETED & Scrub Job FAILED" - else - # No scrub was executed because of the scrub conditions - NOTIFICATION_TITLE="Sync Job COMPLETED" - fi + # Increment check variable + UPD_DEL_CHECK=$((UPD_DEL_CHECK + 1)) else - echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT - _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." - NOTIFICATION_TITLE="Sync Job COMPLETED" + # Update threshold is set but not exceeded. Inform user that UPDATE_COUNT is below threshold and allow to proceed + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> updated files ($UPDATE_COUNT) is below threshold ($UPD_THRESHOLD)." >>$TMP_OUTPUT fi else - NOTIFICATION_TITLE="Sync Job FAILED" + # UPD_THRESHOLD is disabled, so we reflect this information in email and log + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are updated files ($UPDATE_COUNT) but update threshold ($UPD_THRESHOLD) is disabled." >>$TMP_OUTPUT fi - # Send job finished email - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - $NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + # New Logic for v 1.2.0 - check whether delete threshold is enabled or not + if [ $DEL_THRESHOLD -gt 0 ]; then + # Delete threshold is enabled; check if count exceeds threshold + if [ $DEL_COUNT -gt $DEL_THRESHOLD ]; then + # Too many files deleted; inform user and don't proceed with sync + _log "INFO: Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." + _log "INFO: Please run sync manually if this is not an error condition." + echo "Number of deleted files ($DEL_COUNT) exceeded threshold ($DEL_THRESHOLD). NOT proceeding with sync job." >>$TMP_OUTPUT + echo "Please run sync manually if this is not an error condition" >>$TMP_OUTPUT + + # Increment check variable + UPD_DEL_CHECK=$((UPD_DEL_CHECK + 2)) + else + # Delete threshold is set but not exceeded. Inform user that DEL_COUNT is below threshold and allow to proceed + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> deleted files ($DEL_COUNT) is below threshold ($DEL_THRESHOLD)." >>$TMP_OUTPUT + fi + else + # DEL_THRESHOLD is disabled, so we reflect this information in email and log + _log "INFO: Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." + echo "Changes detected [A-$ADD_COUNT,D-$DEL_COUNT,M-$MOVE_COUNT,C-$COPY_COUNT,U-$UPDATE_COUNT] -> there are deleted files ($DEL_COUNT) but delete threshold ($DEL_THRESHOLD) is disabled." >>$TMP_OUTPUT + fi + + # Proceed with sync if no check errors + if [ $UPD_DEL_CHECK -eq 0 ]; then + # start SnapRAID SYNC + _start_sync - # If update threshold exceeded - elif [ $UPD_DEL_CHECK -eq 1 ]; then - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + # If update threshold exceeded + elif [ $UPD_DEL_CHECK -eq 1 ]; then + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + NOTIFICATION_RETURN_CODE=$? - # If delete threshold exceeded - elif [ $UPD_DEL_CHECK -eq 2 ]; then - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + # If delete threshold exceeded + elif [ $UPD_DEL_CHECK -eq 2 ]; then + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + NOTIFICATION_RETURN_CODE=$? - # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) + # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) + else + _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + NOTIFICATION_RETURN_CODE=$? + fi + _check_success $NOTIFICATION_RETURN_CODE && _rmtmp else - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + # NO, so lets log it and exit + _log "INFO: No change detected. Nothing to do" fi - _check_success && _rmtmp -else - # NO, so lets log it and exit - _log "INFO: No change detected. Nothing to do" -fi -_log "INFO: SnapRAID Job ended." -_rmtmp -exit 0 + _log "INFO: SnapRAID Job ended." + _rmtmp + exit 0 +} + +_start_scrub() { + local CURRENT_DATE_EPOCH + local CURRENT_DATE_DATE + local DAYS_DIFF_DATE + local DAYS_DIFF_EPOCH + + SCRUB_SUCCESS="no" + + CURRENT_DATE_EPOCH=$(date +%s) + CURRENT_DATE_DATE=$(epoch_to_date $CURRENT_DATE_EPOCH) + + DAYS_DIFF_EPOCH=$((CURRENT_DATE_EPOCH - $(date_to_epoch $LAST_SCRUB_DATE))) + DAYS_DIFF_DATE=$(seconds_to_days $DAYS_DIFF_EPOCH) + + if [ $LAST_SCRUB_DATE -ne 0 ] && [ $DAYS_DIFF -lt $(($(date_to_seconds $SCRUB_FREQUENCY_IN_DAYS))) ]; then + _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run." + echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run. - $(date)" >>$TMP_OUTPUT + SCRUB_SUCCESS="none" + else + if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ne 0 ]; then + local MAX_NUM_DAYS=$((100 * SCRUB_FREQUENCY_IN_DAYS / SCRUB_PERCENT)) + if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ge $MAX_NUM_DAYS ]; then + # We reached the max num of days, this means we must restart and scrub everything again, + # in an ideal (non-file-changing) scenario we scrubbed all the data + ACCUMULATED_DAYS_SCRUBBED_FILES=0 + fi + fi + _log "INFO: SnapRAID SCRUB Job started" + _log "INFO: ----------------------------------------" + echo "SnapRAID SCRUB Job started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${ACCUMULATED_DAYS_SCRUBBED_FILES} >>$TMP_OUTPUT + + # wait for the job to finish + wait + if [ $? -eq 0 ]; then + SCRUB_SUCCESS="yes" + else + SCRUB_SUCCESS="no" + fi + + ((ACCUMULATED_DAYS_SCRUBBED_FILES += SCRUB_FREQUENCY_IN_DAYS)) + LAST_SCRUB_DATE=$CURRENT_DATE_DATE + + echo "$LAST_SCRUB_DATE" >$TMP_SCRUB_DIR/snapraid_scrub_last_date + echo "$LAST_SCRUB_FREQUENCY" >$TMP_SCRUB_DIR/snapraid_scrub_last_frequency + echo "$LAST_SCRUB_PERCENTAGE" >$TMP_SCRUB_DIR/snapraid_scrub_last_percentage + echo "$ACCUMULATED_DAYS_SCRUBBED_FILES" >$TMP_SCRUB_DIR/snapraid_scrub_accumulated_days + + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SCRUB Job finished" + echo "SnapRAID SCRUB Job finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + fi +} + +# Start SnapRAID Job +_start_job From 75c80e21a6048216bc01d5b1884dda14a0dcc854 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sun, 2 Jun 2024 22:24:03 -0500 Subject: [PATCH 08/15] some changes - store PREHASH option in diff conf file - add option to send SnapRAID commands debug logs as attachments in the Apprise and Ntfy notifications (Ntfy will always be a separate notification, Apprise depends on the service used) - add option to always send the notifications even if no changes were detected by the diff command - renamed options - remove _split_cmd function and do it directly - refactor ntfy and apprise functions to avoid errors about the length of the command - reworked _log() function to always print WARN messages and only print INFO or DEBUG msgs if DEBUG is set to true, and always append the message to the main log file - fixed some logic - simplified code by storing command arguments in variables instead of duplicating the line - removed duped notification prefix --- .../snapraid/files/etc-snapraid-diff_conf.j2 | 5 +- usr/sbin/omv-snapraid-diff | 363 ++++++++++++------ .../confdb/create.d/conf.service.snapraid.sh | 2 + .../datamodels/conf.service.snapraid.json | 8 + .../datamodels/rpc.snapraid.json | 8 + ...-services-snapraid-settings-form-page.yaml | 20 +- 6 files changed, 272 insertions(+), 134 deletions(-) diff --git a/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 b/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 index 5e9a2ed..92a55ed 100644 --- a/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 +++ b/srv/salt/omv/deploy/snapraid/files/etc-snapraid-diff_conf.j2 @@ -6,7 +6,10 @@ SCRUB_FREQUENCY_IN_DAYS={{ config.scrubfreq }} UPD_THRESHOLD={{ config.updthreshold }} DEL_THRESHOLD={{ config.delthreshold }} SCRUB_PERCENT={{ config.scrubpercent }} +{{ 'PREHASH=true' if config.prehash | to_bool else 'PREHASH=false' }} {{ 'USE_APPRISE=true' if config.apprise | to_bool else 'USE_APPRISE=false' }} {{ 'USE_NTFY=true' if config.ntfy | to_bool else 'USE_NTFY=false' }} APPRISE_CMD='{{ config.apprisecmd }}' -NTFY_CMD={{ config.ntfycmd }} +NTFY_CMD="{{ config.ntfycmd }}" +{{ 'SEND_SNAPRAID_CMD_LOGS=true' if config.sendsnaplog | to_bool else 'SEND_SNAPRAID_CMD_LOGS=false' }} +{{ 'ALWAYS_SEND_NOTIFICATION=true' if config.sendnotifalways | to_bool else 'ALWAYS_SEND_NOTIFICATION=false' }} diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 4f4fe54..d1935fb 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -50,15 +50,20 @@ declare -i MOVE_COUNT=0 declare -i COPY_COUNT=0 declare -i UPDATE_COUNT=0 -declare -i PREHASH_VALUE=0 +DIFF_EXECUTED="no" +SYNC_EXECUTED="no" +SCRUB_EXECUTED="no" ARRAY_CONF="${1}" # Set path to snapraid.conf if [ -f "${ARRAY_CONF}" ]; then SNAPRAID_CONF="${ARRAY_CONF}" -else +elif [ -f "/etc/snapraid.conf" ]; then SNAPRAID_CONF="/etc/snapraid.conf" +else + _log "WARN: No SnapRAID configuration file found. Exiting..." + exit 1 fi # the following file is created by the OMV-GUI @@ -67,6 +72,9 @@ SR_DIFF_CONF="/etc/snapraid-diff.conf" NOTIFICATION_SUBJECT_PREFIX="SnapRAID" SNAPRAID_LOG="/var/log/snapraid.log" +SNAPRAID_DIFF_LOG="/var/log/snapraid-diff.log" +SNAPRAID_SYNC_LOG="/var/log/snapraid-sync.log" +SNAPRAID_SCRUB_LOG="/var/log/snapraid-scrub.log" SNAPRAID_NOTIF_LOG="/var/log/snapraid-notif.log" if [ -f "${SNAPRAID_NOTIF_LOG}" ]; then @@ -95,9 +103,8 @@ _send_notification() { local APPRISE_FAIL=false local NTFY_FAIL=false - if [ $SENDMAIL = "false" ] && [ $USE_APPRISE = "false" ] && [ $USE_NTFY = "false" ]; then + if [ "$SENDMAIL" = "false" ] && [ "$USE_APPRISE" = "false" ] && [ "$USE_NTFY" = "false" ]; then _log "INFO: Notifications should be sent here - SENDMAIL, USE_APPRISE and USE_NTFY set to false" - echo "Notifications should be sent here - SENDMAIL, USE_APPRISE and USE_NTFY set to false" return 0 fi @@ -107,34 +114,67 @@ _send_notification() { Yours, SnapRAID-diff script EOF - if [ $SENDMAIL = "true" ]; then + if [ "$USE_APPRISE" = "true" ] || [ "$USE_NTFY" = "true" ]; then + if [ "$SEND_SNAPRAID_CMD_LOGS" = "true" ]; then + SNAP_LOG_ARCHIVE_PATH="$(_zip_logs)" + fi + fi + + if [ "$SENDMAIL" = "true" ]; then _send_email "$1" "$2" - if [ $? -eq 1 ]; then + if [ $? -ne 0 ]; then MAIL_FAIL=true fi fi - if [ $USE_APPRISE = "true" ]; then + if [ "$USE_APPRISE" = "true" ]; then _send_apprise "$1" "$2" "$3" - if [ $? -eq 1 ]; then + if [ $? -ne 0 ]; then APPRISE_FAIL=true fi fi - if [ $USE_NTFY = "true" ]; then + if [ "$USE_NTFY" = "true" ]; then _send_ntfy "$1" "$2" "$3" - if [ $? -eq 1 ]; then + if [ $? -ne 0 ]; then NTFY_FAIL=true fi fi - if [ $MAIL_FAIL = "true" ] || [ $APPRISE_FAIL = "true" ] || [ $NTFY_FAIL = "true" ]; then + if [ "$MAIL_FAIL" = "true" ] || [ "$APPRISE_FAIL" = "true" ] || [ "$NTFY_FAIL" = "true" ]; then return 1 else return 0 fi } +_zip_logs() { + local FILE_NAMES="" + + if [ "$DIFF_EXECUTED" = "yes" ]; then + FILE_NAMES+="$SNAPRAID_DIFF_LOG" + fi + + if [ "$SYNC_EXECUTED" = "yes" ]; then + FILE_NAMES+=" $SNAPRAID_SYNC_LOG" + fi + + if [ "$SCRUB_EXECUTED" = "yes" ]; then + FILE_NAMES+=" $SNAPRAID_SCRUB_LOG" + fi + + if [ "$FILE_NAMES" = "" ]; then + echo "none" + return + fi + + local ARCHIVE_PATH="$TMP_DIR/snap_logs.xz" + + tar -cJf "$ARCHIVE_PATH" $FILE_NAMES + + echo "$ARCHIVE_PATH" +} + _send_email() { ## the below does not work with the internal OMV-Settings # mail -s "$1" -a 'Content-Type: text/plain; charset=utf-8' $MAIL -- -F "$3" -f "$4" < $2 @@ -153,14 +193,13 @@ _send_email() { _send_apprise() { local CMD - local APPRISE_CMD_ARRAY local VERBOSE_LEVEL local APPRISE_RETURN_CODE if $DEBUG; then - VERBOSE_LEVEL=-D + VERBOSE_LEVEL="-vvv" else - VERBOSE_LEVEL=-vv + VERBOSE_LEVEL="-vv" fi echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG @@ -168,21 +207,38 @@ _send_apprise() { echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG if _is_in_path apprise; then - read -r -a APPRISE_CMD_ARRAY <<<"$(_split_cmd $APPRISE_CMD)" - for CMD in "${APPRISE_CMD_ARRAY[@]}"; do - if [ $3 != "" ]; then - apprise $VERBOSE_LEVEL --title "$3 - $1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG 2>&1 - wait - APPRISE_RETURN_CODE=$? - else - apprise $VERBOSE_LEVEL --title "$1" --body "$(cat $2)" "$CMD" >>$SNAPRAID_NOTIF_LOG 2>&1 - wait - APPRISE_RETURN_CODE=$? + local NOTIF_TITLE + + if [ $3 != "" ]; then + NOTIF_TITLE="$3 - $1" + else + NOTIF_TITLE="$1" + fi + + local ATTACH_COMMAND="" + + if [ "$SEND_SNAPRAID_CMD_LOGS" = "true" ] && [ "$SNAP_LOG_ARCHIVE_PATH" != "none" ]; then + ATTACH_COMMAND=("--attach" "$SNAP_LOG_ARCHIVE_PATH") + fi + + local IFS=';' + for CMD in $APPRISE_CMD; do + if $DEBUG; then + local IFS=" " + echo "Apprise command:" >>$SNAPRAID_NOTIF_LOG + echo "apprise $VERBOSE_LEVEL --title \"$NOTIF_TITLE\" \"${ATTACH_COMMAND[*]}\" \"$CMD\" <\"$2\" >>$SNAPRAID_NOTIF_LOG 2>&1" >>$SNAPRAID_NOTIF_LOG + echo "" >>$SNAPRAID_NOTIF_LOG fi + apprise $VERBOSE_LEVEL --title "$NOTIF_TITLE" "${ATTACH_COMMAND[@]}" "$CMD" <"$2" >>$SNAPRAID_NOTIF_LOG 2>&1 + APPRISE_RETURN_CODE=$? done if [ $APPRISE_RETURN_CODE -eq 0 ]; then - _log "INFO: Full status report sent through 'Apprise'" + if [ "$SEND_SNAPRAID_CMD_LOGS" = "true" ]; then + _log "INFO: Full status report and command logs sent through 'Apprise'" + else + _log "INFO: Full status report sent through 'Apprise'" + fi return 0 else _log "WARN: Apprise notification with full status report could not be sent. Please have a look at $SNAPRAID_NOTIF_LOG." @@ -200,30 +256,67 @@ _send_ntfy() { echo "----------------------------------------" >>$SNAPRAID_NOTIF_LOG local VERBOSE_LEVEL - local NTFY_RETURN_CODE + local NTFY_MAIN_RETURN_CODE=-1 + local NTFY_EXTRA_RETURN_CODE=-1 if $DEBUG; then - VERBOSE_LEVEL=debug + VERBOSE_LEVEL="debug" else - VERBOSE_LEVEL=warn + VERBOSE_LEVEL="warn" fi if _is_in_path ntfy; then + + local NOTIF_TITLE + if [ $3 != "" ]; then - ntfy publish --log-level $VERBOSE_LEVEL --title "$3 - $1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1 - wait - NTFY_RETURN_CODE=$? + NOTIF_TITLE="$3 - $1" else - ntfy publish --log-level $VERBOSE_LEVEL --title "$1" --message "$(cat $2)" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1 - wait - NTFY_RETURN_CODE=$? + NOTIF_TITLE="$1" fi - if [ $NTFY_RETURN_CODE -eq 0 ]; then - _log "INFO: Full status report sent through 'Ntfy'" + # Ntfy cli doesn't support grabbing the message from the standard input so we have to send the report as an attachment + if $DEBUG; then + echo "Ntfy command:" >>$SNAPRAID_NOTIF_LOG + echo "ntfy publish --log-level $VERBOSE_LEVEL --title \"$NOTIF_TITLE\" --file \"$2\" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1" >>$SNAPRAID_NOTIF_LOG + echo "" >>$SNAPRAID_NOTIF_LOG + fi + + ntfy publish --log-level $VERBOSE_LEVEL --title "$NOTIF_TITLE" --file "$2" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1 + NTFY_MAIN_RETURN_CODE=$? + + if [ $NTFY_MAIN_RETURN_CODE -eq 0 ] && [ $SEND_SNAPRAID_CMD_LOGS = "true" ] && [ $SNAP_LOG_ARCHIVE_PATH != "none" ]; then + local NOTIF_TITLE="$3 - Command Logs" + + # Ntfy doesn't support sending more than one attachment at the same time, we have to send two notifications. + if $DEBUG; then + echo "Ntfy command:" >>$SNAPRAID_NOTIF_LOG + echo "ntfy publish --log-level $VERBOSE_LEVEL --title \"$NOTIF_TITLE\" --file \"$SNAP_LOG_ARCHIVE_PATH\" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1" >>$SNAPRAID_NOTIF_LOG + echo "" >>$SNAPRAID_NOTIF_LOG + fi + + ntfy publish --log-level $VERBOSE_LEVEL --title "$NOTIF_TITLE" --file "$SNAP_LOG_ARCHIVE_PATH" $NTFY_CMD >>$SNAPRAID_NOTIF_LOG 2>&1 + NTFY_EXTRA_RETURN_CODE=$? + fi + + if [ $NTFY_MAIN_RETURN_CODE -eq 0 ]; then + if [ $NTFY_EXTRA_RETURN_CODE -eq 0 ]; then + _log "INFO: Full status report and SnapRAID command logs sent through 'Ntfy'" + elif [ $NTFY_EXTRA_RETURN_CODE -ne -1 ]; then + _log "WARN: Full status report sent successfully but Ntfy notification with SnapRAID command logs could not be sent. Please have a look at $SNAPRAID_NOTIF_LOG" + return 1 + else + _log "INFO: Full status report sent through 'Ntfy'" + fi + return 0 else _log "WARN: Ntfy notification with full status report could not be sent. Please have a look at $SNAPRAID_NOTIF_LOG." + + if [ "$SEND_SNAPRAID_CMD_LOGS" = "true" ]; then + _log "WARN: Ntfy notification with SnapRAID logs was not sent because main notification failed." + fi + return 1 fi else @@ -254,25 +347,13 @@ _log() { local LOGMESSAGE=${BASH_REMATCH[2]} # the Log-Message } - if $DEBUG; then + if [ "$PRIORITY" = "WARN" ]; then + echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" + elif $DEBUG; then echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" - echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG - else - echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG fi -} - -_split_cmd() { - local CMD_QUOTED="$1" - - local IFS - local CMDS - IFS=';' - read -r -a CMDS <<<"${CMD_QUOTED}" - - IFS=" " - echo "${CMDS[*]}" + echo "$(date '+[%Y-%m-%d %H:%M:%S]') $(basename "$0"): $PRIORITY: '$LOGMESSAGE'" >>$SNAPRAID_LOG } ################################################################ @@ -295,7 +376,7 @@ _rmtmp() { # : 1 = last command ended with an error -> [ $? -gt 1 ] # _check_success() { - if [ $1 = 0 ]; then + if [ $1 -eq 0 ]; then return 0 else return 1 @@ -345,7 +426,7 @@ if [ -f $SR_DIFF_CONF ]; then . $SR_DIFF_CONF else _log "WARN: $SR_DIFF_CONF not found! Exiting ..." - echo "$SR_DIFF_CONF not found! Exiting ..." + exit 1 fi ############################ @@ -389,9 +470,11 @@ TMP_SCRUB_DIR="/root/.snapraid" SNAPRAID_BIN="/usr/bin/snapraid" if [ -z $SNAPRAID_BIN ]; then _log "WARN: snapraid executable not found!" + exit 1 fi if [ ! -x $SNAPRAID_BIN ]; then _log "WARN: snapraid '$SNAPRAID_BIN' not executable! Please do a 'chmod +x $SNAPRAID_BIN'" + exit 1 fi SNAPRAIDVERSION="$(dpkg-query -W -f='${Version}' snapraid)" @@ -411,7 +494,9 @@ CONTENT_FILES_COUNTER=0 # check, if at least one content file is found for SINGLE_CONTENT_FILE in $OMV_CONTENT_FILES; do - if $DEBUG; then _log "DEBUG: found: SINGLE_CONTENT_FILE: $SINGLE_CONTENT_FILE"; fi + if $DEBUG; then + _log "DEBUG: found: SINGLE_CONTENT_FILE: $SINGLE_CONTENT_FILE" + fi if [ -e $SINGLE_CONTENT_FILE ]; then let CONTENT_FILES_COUNTER++ @@ -429,7 +514,9 @@ fi IFS=',' for parity in ${PARITY_FILE}; do if [ -e ${parity} ]; then - if $DEBUG; then _log "DEBUG: found: PARITY_FILE: ${parity}"; fi + if $DEBUG; then + _log "DEBUG: found: PARITY_FILE: ${parity}" + fi else _log "WARN: Parity file (${parity}) not found!" exit 1 @@ -462,7 +549,7 @@ fi if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_freq ]; then LAST_SCRUB_FREQUENCY=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_frequency) - if $LAST_SCRUB_FREQUENCY != $SCRUB_FREQUENCY_IN_DAYS; then + if [ $LAST_SCRUB_FREQUENCY -ne $SCRUB_FREQUENCY_IN_DAYS ]; then RESET_SCRUB_CALC="true" fi fi @@ -471,7 +558,7 @@ LAST_SCRUB_FREQUENCY=$SCRUB_FREQUENCY_IN_DAYS if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_percentage ]; then LAST_SCRUB_PERCENTAGE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_percentage) - if $LAST_SCRUB_PERCENTAGE != $SCRUB_PERCENT; then + if [ $LAST_SCRUB_PERCENTAGE -ne $SCRUB_PERCENT ]; then RESET_SCRUB_CALC="true" fi fi @@ -486,7 +573,7 @@ else ACCUMULATED_DAYS_SCRUBBED_FILES=0 fi -if [ $RESET_SCRUB_CALC = "true" ]; then +if [ "$RESET_SCRUB_CALC" = "true" ]; then # If the user changes either the scrub frequency or percentage then we start the scrub from 0, # resetting the current progress. ACCUMULATED_DAYS_SCRUBBED_FILES=0 @@ -515,9 +602,10 @@ fi _start_job() { ################################################################ ## Start of the script - _log "INFO: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" + _log "INFO: ========================================" _log "INFO: SnapRAID Job started - Script-version: $SCRIPTVERSION" _log "INFO: SnapRAID-Version: $SNAPRAIDVERSION" + _log "INFO: ========================================" _add_mail_header # all info above is in that function _check_disks @@ -529,7 +617,12 @@ _start_job() { } _check_disks() { + _log "INFO: ----------------------------------------" _log "INFO: Checking 'disks' in '$SNAPRAID_CONF'" + _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT + echo "Checking 'disks' in '$SNAPRAID_CONF'" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT local OMV_DISK local NOTIFICATION_RETURN_CODE @@ -560,91 +653,47 @@ _check_disks() { echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >>$TMP_OUTPUT echo "Lastly, use 'snapraid sync' to update the parity files." >>$TMP_OUTPUT - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" NOTIFICATION_RETURN_CODE=$? _check_success $NOTIFICATION_RETURN_CODE && _rmtmp _log "WARN: script exits here due to missing disk ..." exit 1 fi done -} -_start_sync() { - _log "INFO: " - _log "INFO: SnapRAID SYNC Job started" _log "INFO: ----------------------------------------" - echo " " >>$TMP_OUTPUT - echo "SnapRAID SYNC Job started - $(date)" >>$TMP_OUTPUT - echo "----------------------------------------" >>$TMP_OUTPUT - - local SYNC_EXIT_CODE - local NOTIFICATION_TITLE - - # we should eventually grep % - if [ ${PREHASH_VALUE} -eq 1 ]; then - /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} --pre-hash >>$TMP_OUTPUT - - #wait for the job to finish - wait - SYNC_EXIT_CODE=$? - else - /usr/bin/snapraid sync --conf ${SNAPRAID_CONF} >>$TMP_OUTPUT - - #wait for the job to finish - wait - SYNC_EXIT_CODE=$? - fi - + _log "INFO: Finished checking 'disks'" _log "INFO: ----------------------------------------" - _log "INFO: SnapRAID SYNC Job finished" - echo "SnapRAID SYNC Job finished - $(date)" >>$TMP_OUTPUT echo "----------------------------------------" >>$TMP_OUTPUT - - # start SnapRAID SCRUB - if [ $SYNC_EXIT_CODE -eq 0 ]; then - if [ $RUN_SCRUB = "true" ]; then - _start_scrub - if [ $SCRUB_SUCCESS = "yes" ]; then - NOTIFICATION_TITLE="Sync & Scrub Job COMPLETED" - elif [ $SCRUB_SUCCESS = "no" ]; then - NOTIFICATION_TITLE="Sync job COMPLETED & Scrub Job FAILED" - else - # No scrub was executed because of the scrub conditions - NOTIFICATION_TITLE="Sync Job COMPLETED" - fi - else - echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT - _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." - NOTIFICATION_TITLE="Sync Job COMPLETED" - fi - else - NOTIFICATION_TITLE="Sync Job FAILED" - fi - - # Send job finished email - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - $NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + echo "Finished checking 'disks'" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT } _start_diff() { + _log "INFO: ----------------------------------------" _log "INFO: SnapRAID DIFF started" + _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT echo "SnapRAID DIFF started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT - local UPD_THRESHOLD - local DEL_THRESHOLD local NOTIFICATION_RETURN_CODE local line # run diff: # output: stdout: WARNINGS only (filtering 'Ignoring mount point') and $TMP_OUTPUT (all WARNINGS) - /usr/bin/snapraid diff --conf ${SNAPRAID_CONF} 2>&1 | tee -a $TMP_OUTPUT | grep WARNING | grep -v 'WARNING! Ignoring mount point' | while read line; do + $SNAPRAID_BIN diff --conf ${SNAPRAID_CONF} --log "$SNAPRAID_DIFF_LOG" 2>&1 | tee -a $TMP_OUTPUT | grep WARNING | grep -v 'WARNING! Ignoring mount point' | while read line; do _log "WARN: $line" done # wait for the above cmd to finish wait + DIFF_EXECUTED=yes + _log "INFO: ----------------------------------------" _log "INFO: SnapRAID DIFF finished" _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT echo "SnapRAID DIFF finished - $(date)" >>$TMP_OUTPUT echo "----------------------------------------" >>$TMP_OUTPUT @@ -658,18 +707,13 @@ _start_diff() { _log "INFO: SUMMARY of changes since last sync:" _log "INFO: Added: [$ADD_COUNT] - Deleted: [$DEL_COUNT] - Moved: [$MOVE_COUNT] - Copied: [$COPY_COUNT] - Updated: [$UPDATE_COUNT]" - # set pre-hash option - PREHASH_VALUE=$(omv_config_get "/config/services/snapraid/prehash") if $DEBUG; then - _log "DEBUG: Pre-Hash option: ${PREHASH_VALUE}" + _log "DEBUG: Pre-Hash option: '$PREHASH'" fi # check if files have changed if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $COPY_COUNT -gt 0 ] || [ $UPDATE_COUNT -gt 0 ]; then - UPD_THRESHOLD=$(omv_config_get "/config/services/snapraid/updthreshold") - DEL_THRESHOLD=$(omv_config_get "/config/services/snapraid/delthreshold") - # New Logic for v 1.2.0 - check whether update threshold is enabled or not if [ $UPD_THRESHOLD -gt 0 ]; then # Update threshold is enabled; check if count exceeds threshold @@ -723,23 +767,27 @@ _start_diff() { # If update threshold exceeded elif [ $UPD_DEL_CHECK -eq 1 ]; then - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" NOTIFICATION_RETURN_CODE=$? # If delete threshold exceeded elif [ $UPD_DEL_CHECK -eq 2 ]; then - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" NOTIFICATION_RETURN_CODE=$? # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) else - _send_notification "$NOTIFICATION_SUBJECT_PREFIX - WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" NOTIFICATION_RETURN_CODE=$? fi _check_success $NOTIFICATION_RETURN_CODE && _rmtmp else # NO, so lets log it and exit _log "INFO: No change detected. Nothing to do" + + if [ "$ALWAYS_SEND_NOTIFICATION" = "true" ]; then + _send_notification "No changes on data disks" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + fi fi _log "INFO: SnapRAID Job ended." @@ -747,6 +795,59 @@ _start_diff() { exit 0 } +_start_sync() { + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SYNC Job started" + _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT + echo "SnapRAID SYNC Job started - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + + local SYNC_EXIT_CODE + local NOTIFICATION_TITLE + local PREHASH_COMMAND="" + + # we should eventually grep % + if [ "$PREHASH" = "true" ]; then + PREHASH_COMMAND="--pre-hash" + fi + + $SNAPRAID_BIN sync --conf ${SNAPRAID_CONF} $PREHASH_COMMAND --log "$SNAPRAID_SYNC_LOG" >>$TMP_OUTPUT + SYNC_EXIT_CODE=$? + SYNC_EXECUTED="yes" + + _log "INFO: ----------------------------------------" + _log "INFO: SnapRAID SYNC Job finished" + _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT + echo "SnapRAID SYNC Job finished - $(date)" >>$TMP_OUTPUT + echo "----------------------------------------" >>$TMP_OUTPUT + + # start SnapRAID SCRUB + if [ $SYNC_EXIT_CODE -eq 0 ]; then + if [ $RUN_SCRUB = "true" ]; then + _start_scrub + if [ $SCRUB_SUCCESS = "yes" ]; then + NOTIFICATION_TITLE="Sync & Scrub Job COMPLETED" + elif [ $SCRUB_SUCCESS = "no" ]; then + NOTIFICATION_TITLE="Sync job COMPLETED & Scrub Job FAILED" + else + # No scrub was executed because of the scrub conditions + NOTIFICATION_TITLE="Sync Job COMPLETED" + fi + else + echo "Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." >>$TMP_OUTPUT + _log "INFO: Array scrubbing is not enabled. If you would like to enable it, please set the RUN_SCRUB option to true." + NOTIFICATION_TITLE="Sync Job COMPLETED" + fi + else + NOTIFICATION_TITLE="Sync Job FAILED" + fi + + # Send job finished email + _send_notification "$NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" +} + _start_scrub() { local CURRENT_DATE_EPOCH local CURRENT_DATE_DATE @@ -774,20 +875,24 @@ _start_scrub() { ACCUMULATED_DAYS_SCRUBBED_FILES=0 fi fi + + _log "INFO: ----------------------------------------" _log "INFO: SnapRAID SCRUB Job started" _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT echo "SnapRAID SCRUB Job started - $(date)" >>$TMP_OUTPUT echo "----------------------------------------" >>$TMP_OUTPUT - /usr/bin/snapraid scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${ACCUMULATED_DAYS_SCRUBBED_FILES} >>$TMP_OUTPUT - # wait for the job to finish - wait + $SNAPRAID_BIN scrub --conf ${SNAPRAID_CONF} -p ${SCRUB_PERCENT} -o ${ACCUMULATED_DAYS_SCRUBBED_FILES} --log "$SNAPRAID_SCRUB_LOG" >>$TMP_OUTPUT + if [ $? -eq 0 ]; then SCRUB_SUCCESS="yes" else SCRUB_SUCCESS="no" fi + SCRUB_EXECUTED="yes" + ((ACCUMULATED_DAYS_SCRUBBED_FILES += SCRUB_FREQUENCY_IN_DAYS)) LAST_SCRUB_DATE=$CURRENT_DATE_DATE @@ -798,6 +903,8 @@ _start_scrub() { _log "INFO: ----------------------------------------" _log "INFO: SnapRAID SCRUB Job finished" + _log "INFO: ----------------------------------------" + echo "----------------------------------------" >>$TMP_OUTPUT echo "SnapRAID SCRUB Job finished - $(date)" >>$TMP_OUTPUT echo "----------------------------------------" >>$TMP_OUTPUT fi diff --git a/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh b/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh index 8b2b55a..6f8a758 100755 --- a/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh +++ b/usr/share/openmediavault/confdb/create.d/conf.service.snapraid.sh @@ -39,6 +39,8 @@ if ! omv_config_exists "${SERVICE_XPATH}"; then omv_config_add_key "${SERVICE_XPATH}" "apprisecmd" "" omv_config_add_key "${SERVICE_XPATH}" "ntfy" "0" omv_config_add_key "${SERVICE_XPATH}" "ntfycmd" "" + omv_config_add_key "${SERVICE_XPATH}" "sendsnaplog" "0" + omv_config_add_key "${SERVICE_XPATH}" "sendnotifalways" "0" omv_config_add_key "${SERVICE_XPATH}" "runscrub" "1" omv_config_add_key "${SERVICE_XPATH}" "scrubfreq" "7" omv_config_add_key "${SERVICE_XPATH}" "updthreshold" "0" diff --git a/usr/share/openmediavault/datamodels/conf.service.snapraid.json b/usr/share/openmediavault/datamodels/conf.service.snapraid.json index f4e40ed..890e565 100644 --- a/usr/share/openmediavault/datamodels/conf.service.snapraid.json +++ b/usr/share/openmediavault/datamodels/conf.service.snapraid.json @@ -57,6 +57,14 @@ "ntfycmd": { "type": "string" }, + "sendsnaplog": { + "type": "boolean", + "default": false + }, + "sendnotifalways": { + "type": "boolean", + "default": false + }, "runscrub": { "type": "boolean", "default": true diff --git a/usr/share/openmediavault/datamodels/rpc.snapraid.json b/usr/share/openmediavault/datamodels/rpc.snapraid.json index c1b309c..6dd4508 100644 --- a/usr/share/openmediavault/datamodels/rpc.snapraid.json +++ b/usr/share/openmediavault/datamodels/rpc.snapraid.json @@ -75,6 +75,14 @@ "type": "string", "required": true }, + "sendsnaplog": { + "type": "boolean", + "required": true + }, + "sendnotifalways": { + "type": "boolean", + "required": true + }, "runscrub": { "type": "boolean", "required": true diff --git a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml index 8d4fb04..fcfcea6 100644 --- a/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml +++ b/usr/share/openmediavault/workbench/component.d/omv-services-snapraid-settings-form-page.yaml @@ -58,11 +58,11 @@ data: value: false - type: checkbox name: sendmail - label: _("Send mail") + label: _("Send Mail notification") value: true - type: checkbox name: apprise - label: _("Use Apprise") + label: _("Send Apprise notification") value: false - type: textInput name: apprisecmd @@ -77,19 +77,29 @@ data: errorData: _("The field is invalid, it can't contain quotes and the path to the config file and URL can't contain spaces. Path to config file must be full (start with '/').") - type: checkbox name: ntfy - label: _("Use Ntfy") + label: _("Send Ntfy notification") value: false - type: textInput name: ntfycmd - label: _("Ntfy config command or host and/or topic.") + label: _("Ntfy config command or host and/or topic. Report will be sent as an attachment.") hint: _("E.g. ntfy.local/myTopic | myTopic | --config /someConfig.yml") - value: "OMV_SnapRAID" + value: "" trim: true language: shell validators: pattern: pattern: "^(([\\w\\.]+\\/)?[-\\w]{1,64}|--config\\s\\/[^\"'\\s\\|\\&]+)$" errorData: _("The field can only contain a config command, a valid topic name or host and topic (don't add https:// at the start). No quotes or spaces are allowed. Path to config file must be full (start with '/').") + - type: checkbox + name: sendsnaplog + label: _("Send SnapRAID command logs") + value: true + hint: _("Also send a compressed file with the SnapRAID commands (sync, diff, scrub) debug logs as attachment for Apprise and Ntfy notifications.") + - type: checkbox + name: sendnotifalways + label: _("Always send notifications") + value: false + hint: _("Send notifications even if there are no data changes.") - type: checkbox name: runscrub label: _("Run scrub") From d69f264763a9321622cee8e4228d6a4b43a859d9 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Sun, 2 Jun 2024 23:46:58 -0500 Subject: [PATCH 09/15] move everything to functions & improved debug logging now everything except some variables is inside a function improved debug logging to make it more readable --- usr/sbin/omv-snapraid-diff | 444 ++++++++++++++++++++----------------- 1 file changed, 238 insertions(+), 206 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index d1935fb..5ceec6e 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -40,35 +40,6 @@ SCRIPTVERSION="1.3.5" # needed for omv_config_get command . /usr/share/openmediavault/scripts/helper-functions -# Set error count variable -# 0 for no thresholds exceeded; 1 for update threshold exceeded; 2 for delete threshold exceeded; 3 for both exceeded -declare -i UPD_DEL_CHECK=0 - -declare -i DEL_COUNT=0 -declare -i ADD_COUNT=0 -declare -i MOVE_COUNT=0 -declare -i COPY_COUNT=0 -declare -i UPDATE_COUNT=0 - -DIFF_EXECUTED="no" -SYNC_EXECUTED="no" -SCRUB_EXECUTED="no" - -ARRAY_CONF="${1}" - -# Set path to snapraid.conf -if [ -f "${ARRAY_CONF}" ]; then - SNAPRAID_CONF="${ARRAY_CONF}" -elif [ -f "/etc/snapraid.conf" ]; then - SNAPRAID_CONF="/etc/snapraid.conf" -else - _log "WARN: No SnapRAID configuration file found. Exiting..." - exit 1 -fi - -# the following file is created by the OMV-GUI -SR_DIFF_CONF="/etc/snapraid-diff.conf" - NOTIFICATION_SUBJECT_PREFIX="SnapRAID" SNAPRAID_LOG="/var/log/snapraid.log" @@ -77,9 +48,12 @@ SNAPRAID_SYNC_LOG="/var/log/snapraid-sync.log" SNAPRAID_SCRUB_LOG="/var/log/snapraid-scrub.log" SNAPRAID_NOTIF_LOG="/var/log/snapraid-notif.log" -if [ -f "${SNAPRAID_NOTIF_LOG}" ]; then - rm -f "$SNAPRAID_NOTIF_LOG" -fi +## INTERNAL TEMP VARS ## +TMP_DIR="/tmp/snapraid" +TMP_OUTPUT="$TMP_DIR/snapraid.out" +TMP_SCRUB_DIR="/root/.snapraid" + +SNAPRAID_BIN="/usr/bin/snapraid" ################################################################ ##### Functions @@ -170,7 +144,7 @@ _zip_logs() { local ARCHIVE_PATH="$TMP_DIR/snap_logs.xz" - tar -cJf "$ARCHIVE_PATH" $FILE_NAMES + tar -cJf "$ARCHIVE_PATH" $FILE_NAMES &>/dev/null echo "$ARCHIVE_PATH" } @@ -235,7 +209,7 @@ _send_apprise() { if [ $APPRISE_RETURN_CODE -eq 0 ]; then if [ "$SEND_SNAPRAID_CMD_LOGS" = "true" ]; then - _log "INFO: Full status report and command logs sent through 'Apprise'" + _log "INFO: Full status report and SnapRAID command logs sent through 'Apprise'" else _log "INFO: Full status report sent through 'Apprise'" fi @@ -421,192 +395,251 @@ seconds_to_days() { ##### MAIN BODY of the script ################################################################ -# read config -if [ -f $SR_DIFF_CONF ]; then - . $SR_DIFF_CONF -else - _log "WARN: $SR_DIFF_CONF not found! Exiting ..." - exit 1 -fi - -############################ -# Mail settings: -#primary and secondary recipients -OMV_MAIL_primarymail=$(omv_config_get "//system/email/primaryemail") -OMV_MAIL_secondarymail=$(omv_config_get "//system/email/secondaryemail") - -if $DEBUG; then - _log "DEBUG: OMV_MAIL_primarymail $OMV_MAIL_primarymail" - _log "DEBUG: OMV_MAIL_secondarymail $OMV_MAIL_secondarymail" -fi - -# structure in /etc/openmediavault/config.xml - -# -# 1 -# mail.blablabla.com -# 25 -# 0 -# server@blablabla.com -# -# 0 -# -# -# -# someuser@blablabla.net -# -# - -# read CONTENT and PARITY from $SNAPRAID_CONF -OMV_CONTENT_FILES="$(awk '$1 == "content" { print $2 }' ${SNAPRAID_CONF})" -PARITY_FILE="$(awk '$1 == "parity" { print $2 }' ${SNAPRAID_CONF})" -OMV_SNAPRAID_DISKS="$(awk '$1 == "data" { print $3 }' ${SNAPRAID_CONF})" +_pre_work() { + _log "INFO: ========================================" + _log "INFO: SnapRAID Job started - Script-version: $SCRIPTVERSION" + _log "INFO: SnapRAID-Version: $SNAPRAIDVERSION" + _log "INFO: ========================================" -## INTERNAL TEMP VARS ## -TMP_DIR="/tmp/snapraid" -TMP_OUTPUT="$TMP_DIR/snapraid.out" -TMP_SCRUB_DIR="/root/.snapraid" + _add_mail_header # all info above is in that function -SNAPRAID_BIN="/usr/bin/snapraid" -if [ -z $SNAPRAID_BIN ]; then - _log "WARN: snapraid executable not found!" - exit 1 -fi -if [ ! -x $SNAPRAID_BIN ]; then - _log "WARN: snapraid '$SNAPRAID_BIN' not executable! Please do a 'chmod +x $SNAPRAID_BIN'" - exit 1 -fi + # Set error count variable + # 0 for no thresholds exceeded; 1 for update threshold exceeded; 2 for delete threshold exceeded; 3 for both exceeded + declare -gi UPD_DEL_CHECK=0 + + declare -gi DEL_COUNT=0 + declare -gi ADD_COUNT=0 + declare -gi MOVE_COUNT=0 + declare -gi COPY_COUNT=0 + declare -gi UPDATE_COUNT=0 + + DIFF_EXECUTED="no" + SYNC_EXECUTED="no" + SCRUB_EXECUTED="no" + + local ARRAY_CONF + local OMV_CONTENT_FILES + local PARITY_FILE + local SR_DIFF_CONF + local SINGLE_CONTENT_FILE + local RESET_SCRUB_CALC + + ARRAY_CONF="${1}" + + # Set path to snapraid.conf + if [ -f "${ARRAY_CONF}" ]; then + SNAPRAID_CONF="${ARRAY_CONF}" + elif [ -f "/etc/snapraid.conf" ]; then + SNAPRAID_CONF="/etc/snapraid.conf" + else + _log "WARN: No SnapRAID configuration file found. Exiting..." + exit 1 + fi -SNAPRAIDVERSION="$(dpkg-query -W -f='${Version}' snapraid)" + # the following file is created by the OMV-GUI + SR_DIFF_CONF="/etc/snapraid-diff.conf" -# check tmp-dirs -if [ ! -d $TMP_DIR ]; then - mkdir $TMP_DIR -fi + if [ -f "${SNAPRAID_NOTIF_LOG}" ]; then + rm -f "$SNAPRAID_NOTIF_LOG" + fi -if [ -f $TMP_OUTPUT ]; then - rm $TMP_OUTPUT && _log "INFO: '$TMP_OUTPUT' deleted - creating a new one" -fi + # read config + if [ -f $SR_DIFF_CONF ]; then + . $SR_DIFF_CONF + else + _log "WARN: $SR_DIFF_CONF not found! Exiting ..." + exit 1 + fi -#sanity check first to make sure we can access the content and parity files -CONTENT_FILES_COUNTER=0 + ############################ + # Mail settings: + # primary and secondary recipients + OMV_MAIL_primarymail=$(omv_config_get "//system/email/primaryemail") + OMV_MAIL_secondarymail=$(omv_config_get "//system/email/secondaryemail") + + # structure in /etc/openmediavault/config.xml + + # + # 1 + # mail.blablabla.com + # 25 + # 0 + # server@blablabla.com + # + # 0 + # + # + # + # someuser@blablabla.net + # + # + + # read CONTENT and PARITY from $SNAPRAID_CONF + + OMV_CONTENT_FILES="$(awk '$1 == "content" { print $2 }' ${SNAPRAID_CONF})" + PARITY_FILE="$(awk '$1 == "parity" { print $2 }' ${SNAPRAID_CONF})" + OMV_SNAPRAID_DISKS="$(awk '$1 == "data" { print $3 }' ${SNAPRAID_CONF})" + + if [ -z $SNAPRAID_BIN ]; then + _log "WARN: snapraid executable not found!" + exit 1 + fi + if [ ! -x $SNAPRAID_BIN ]; then + _log "WARN: snapraid '$SNAPRAID_BIN' not executable! Please do a 'chmod +x $SNAPRAID_BIN'" + exit 1 + fi -# check, if at least one content file is found -for SINGLE_CONTENT_FILE in $OMV_CONTENT_FILES; do + SNAPRAIDVERSION="$(dpkg-query -W -f='${Version}' snapraid)" - if $DEBUG; then - _log "DEBUG: found: SINGLE_CONTENT_FILE: $SINGLE_CONTENT_FILE" + # check tmp-dirs + if [ ! -d $TMP_DIR ]; then + mkdir $TMP_DIR fi - if [ -e $SINGLE_CONTENT_FILE ]; then - let CONTENT_FILES_COUNTER++ - continue - else - _log "WARN: Content file ($SINGLE_CONTENT_FILE) not found!" + if [ -f $TMP_OUTPUT ]; then + rm $TMP_OUTPUT && _log "INFO: '$TMP_OUTPUT' deleted - creating a new one" fi -done -if [ $CONTENT_FILES_COUNTER -eq 0 ]; then - _log "WARN: No Content files found! Exiting ..." - exit 1 -fi + #sanity check first to make sure we can access the content and parity files + local CONTENT_FILES_COUNTER=0 -IFS=',' -for parity in ${PARITY_FILE}; do - if [ -e ${parity} ]; then + # check, if at least one content file is found + local IFS=$'\n' + for SINGLE_CONTENT_FILE in $OMV_CONTENT_FILES; do if $DEBUG; then - _log "DEBUG: found: PARITY_FILE: ${parity}" + _log "DEBUG: found: SINGLE_CONTENT_FILE: $SINGLE_CONTENT_FILE" fi - else - _log "WARN: Parity file (${parity}) not found!" + + if [ -e $SINGLE_CONTENT_FILE ]; then + let CONTENT_FILES_COUNTER++ + continue + else + _log "WARN: Content file ($SINGLE_CONTENT_FILE) not found!" + fi + done + + if [ $CONTENT_FILES_COUNTER -eq 0 ]; then + _log "WARN: No Content files found! Exiting ..." exit 1 fi -done - -### Section to support SnapRAID Scrub ### -mkdir -p $TMP_SCRUB_DIR - -# define Variables, if not set in snapraid-diff -if [ -z ${SCRUB_PERCENT} ]; then - SCRUB_PERCENT=100 - _log "INFO: SCRUB_PERCENT not set in '$SR_DIFF_CONF'" - _log "INFO: set it to $SCRUB_PERCENT" -fi -if [ -z ${SCRUB_FREQUENCY_IN_DAYS} ]; then - SCRUB_FREQUENCY_IN_DAYS=0 - _log "INFO: SCRUB_FREQUENCY_IN_DAYS not set in '$SR_DIFF_CONF'" - _log "INFO: set it to $SCRUB_FREQUENCY_IN_DAYS" -fi - -# for scrub -RESET_SCRUB_CALC="false" - -if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_date ]; then - LAST_SCRUB_DATE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_date) -else - LAST_SCRUB_DATE=0 -fi - -if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_freq ]; then - LAST_SCRUB_FREQUENCY=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_frequency) - if [ $LAST_SCRUB_FREQUENCY -ne $SCRUB_FREQUENCY_IN_DAYS ]; then - RESET_SCRUB_CALC="true" - fi -fi - -LAST_SCRUB_FREQUENCY=$SCRUB_FREQUENCY_IN_DAYS - -if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_percentage ]; then - LAST_SCRUB_PERCENTAGE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_percentage) - if [ $LAST_SCRUB_PERCENTAGE -ne $SCRUB_PERCENT ]; then - RESET_SCRUB_CALC="true" - fi -fi - -LAST_SCRUB_PERCENTAGE=$SCRUB_PERCENT - -# We'll store the accumulated days of the scrubbed files, -# that way if we use 10% every 7 days, we are sure in 70 days the whole array is scrubbed -if [ -f $TMP_SCRUB_DIR/snapraid_scrub_accumulated_days ]; then - ACCUMULATED_DAYS_SCRUBBED_FILES=$(cat $TMP_SCRUB_DIR/snapraid_scrub_accumulated_days) -else - ACCUMULATED_DAYS_SCRUBBED_FILES=0 -fi - -if [ "$RESET_SCRUB_CALC" = "true" ]; then - # If the user changes either the scrub frequency or percentage then we start the scrub from 0, - # resetting the current progress. - ACCUMULATED_DAYS_SCRUBBED_FILES=0 - LAST_SCRUB_DATE=0 -fi -### End Scrub Section ### - -if $DEBUG; then - _log "DEBUG: OMV_CONTENT_FILES: $OMV_CONTENT_FILES" - _log "DEBUG: PARITY_FILE: $PARITY_FILE" - _log "DEBUG: OMV_SNAPRAID_DISKS: $OMV_SNAPRAID_DISKS" - _log "DEBUG: TMP_OUTPUT: $TMP_OUTPUT" - _log "DEBUG: RUN_SCRUB: $RUN_SCRUB" - _log "DEBUG: SCRUB_FREQUENCY_IN_DAYS: $SCRUB_FREQUENCY_IN_DAYS" - _log "DEBUG: SCRUB_PERCENT: $SCRUB_PERCENT" - _log "DEBUG: DEL_THRESHOLD: $DEL_THRESHOLD" - _log "DEBUG: UPD_THRESHOLD: $UPD_THRESHOLD" - _log "DEBUG: APPRISE_CMD: ${APPRISE_CMD}" - _log "DEBUG: NTFY_CMD: $NTFY_CMD" - _log "DEBUG: LAST_SCRUB_DATE: $LAST_SCRUB_DATE" - _log "DEBUG: LAST_SCRUB_FREQUENCY: $LAST_SCRUB_FREQUENCY" - _log "DEBUG: LAST_SCRUB_PERCENTAGE: $LAST_SCRUB_PERCENTAGE" - _log "DEBUG: ACCUMULATED_DAYS_SCRUBBED_FILES: $ACCUMULATED_DAYS_SCRUBBED_FILES" -fi - -_start_job() { + + IFS=',' + local parity + for parity in ${PARITY_FILE}; do + if [ -e ${parity} ]; then + if $DEBUG; then + _log "DEBUG: found: PARITY_FILE: ${parity}" + fi + else + _log "WARN: Parity file (${parity}) not found!" + exit 1 + fi + done + + ### Section to support SnapRAID Scrub ### + mkdir -p $TMP_SCRUB_DIR + + # define Variables, if not set in snapraid-diff + if [ -z ${SCRUB_PERCENT} ]; then + SCRUB_PERCENT=100 + _log "INFO: SCRUB_PERCENT not set in '$SR_DIFF_CONF'" + _log "INFO: set it to $SCRUB_PERCENT" + fi + if [ -z ${SCRUB_FREQUENCY_IN_DAYS} ]; then + SCRUB_FREQUENCY_IN_DAYS=0 + _log "INFO: SCRUB_FREQUENCY_IN_DAYS not set in '$SR_DIFF_CONF'" + _log "INFO: set it to $SCRUB_FREQUENCY_IN_DAYS" + fi + + # for scrub + RESET_SCRUB_CALC="false" + + if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_date ]; then + LAST_SCRUB_DATE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_date) + else + LAST_SCRUB_DATE=0 + fi + + if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_freq ]; then + LAST_SCRUB_FREQUENCY=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_frequency) + if [ $LAST_SCRUB_FREQUENCY -ne $SCRUB_FREQUENCY_IN_DAYS ]; then + RESET_SCRUB_CALC="true" + fi + fi + + LAST_SCRUB_FREQUENCY=$SCRUB_FREQUENCY_IN_DAYS + + if [ -f $TMP_SCRUB_DIR/snapraid_scrub_last_percentage ]; then + LAST_SCRUB_PERCENTAGE=$(cat $TMP_SCRUB_DIR/snapraid_scrub_last_percentage) + if [ $LAST_SCRUB_PERCENTAGE -ne $SCRUB_PERCENT ]; then + RESET_SCRUB_CALC="true" + fi + fi + + LAST_SCRUB_PERCENTAGE=$SCRUB_PERCENT + + # We'll store the accumulated days of the scrubbed files, + # that way if we use 10% every 7 days, we are sure in 70 days the whole array is scrubbed + if [ -f $TMP_SCRUB_DIR/snapraid_scrub_accumulated_days ]; then + ACCUMULATED_DAYS_SCRUBBED_FILES=$(cat $TMP_SCRUB_DIR/snapraid_scrub_accumulated_days) + else + ACCUMULATED_DAYS_SCRUBBED_FILES=0 + fi + + if [ "$RESET_SCRUB_CALC" = "true" ]; then + # If the user changes either the scrub frequency or percentage then we start the scrub from 0, + # resetting the current progress. + ACCUMULATED_DAYS_SCRUBBED_FILES=0 + LAST_SCRUB_DATE=0 + fi + ### End Scrub Section ### + + if $DEBUG; then + _log "INFO: ----------------------------------------" + _log "INFO: Configuration Variables" + _log "INFO: ----------------------------------------" + + local IFS=$'\n' + local CONTENT_FILE + for CONTENT_FILE in $OMV_CONTENT_FILES; do + _log "DEBUG: OMV_CONTENT_FILES: $CONTENT_FILE" + done + + local PAR_FILE + for PAR_FILE in $PARITY_FILE; do + _log "DEBUG: PARITY_FILE: $PAR_FILE" + done + + local DATA_DISK + for DATA_DISK in $OMV_SNAPRAID_DISKS; do + _log "DEBUG: OMV_SNAPRAID_DISKS: $DATA_DISK" + done + _log "DEBUG: TMP_OUTPUT: $TMP_OUTPUT" + _log "DEBUG: RUN_SCRUB: $RUN_SCRUB" + _log "DEBUG: SCRUB_FREQUENCY_IN_DAYS: $SCRUB_FREQUENCY_IN_DAYS" + _log "DEBUG: SCRUB_PERCENT: $SCRUB_PERCENT" + _log "DEBUG: DEL_THRESHOLD: $DEL_THRESHOLD" + _log "DEBUG: UPD_THRESHOLD: $UPD_THRESHOLD" + _log "DEBUG: APPRISE_CMD: ${APPRISE_CMD}" + _log "DEBUG: NTFY_CMD: $NTFY_CMD" + _log "DEBUG: LAST_SCRUB_DATE: $LAST_SCRUB_DATE" + _log "DEBUG: LAST_SCRUB_FREQUENCY: $LAST_SCRUB_FREQUENCY" + _log "DEBUG: LAST_SCRUB_PERCENTAGE: $LAST_SCRUB_PERCENTAGE" + _log "DEBUG: ACCUMULATED_DAYS_SCRUBBED_FILES: $ACCUMULATED_DAYS_SCRUBBED_FILES" + _log "DEBUG: OMV_MAIL_primarymail $OMV_MAIL_primarymail" + _log "DEBUG: OMV_MAIL_secondarymail $OMV_MAIL_secondarymail" + _log "DEBUG: PREHASH: $PREHASH" + _log "DEBUG: SEND_SNAPRAID_CMD_LOGS: $SEND_SNAPRAID_CMD_LOGS" + _log "DEBUG: ALWAYS_SEND_NOTIFICATION: $ALWAYS_SEND_NOTIFICATION" + _log "DEBUG: OMV_MAIL_sender: $OMV_MAIL_sender" + + _log "INFO: ----------------------------------------" + fi +} + +_start() { ################################################################ ## Start of the script - _log "INFO: ========================================" - _log "INFO: SnapRAID Job started - Script-version: $SCRIPTVERSION" - _log "INFO: SnapRAID-Version: $SNAPRAIDVERSION" - _log "INFO: ========================================" - _add_mail_header # all info above is in that function _check_disks @@ -629,6 +662,7 @@ _check_disks() { ### check if all hdds in the config are there + local IFS=$'\n' for OMV_DISK in $OMV_SNAPRAID_DISKS; do if $DEBUG; then _log "DEBUG: found: OMV_DISK: $OMV_DISK" @@ -707,10 +741,6 @@ _start_diff() { _log "INFO: SUMMARY of changes since last sync:" _log "INFO: Added: [$ADD_COUNT] - Deleted: [$DEL_COUNT] - Moved: [$MOVE_COUNT] - Copied: [$COPY_COUNT] - Updated: [$UPDATE_COUNT]" - if $DEBUG; then - _log "DEBUG: Pre-Hash option: '$PREHASH'" - fi - # check if files have changed if [ $DEL_COUNT -gt 0 ] || [ $ADD_COUNT -gt 0 ] || [ $MOVE_COUNT -gt 0 ] || [ $COPY_COUNT -gt 0 ] || [ $UPDATE_COUNT -gt 0 ]; then @@ -910,5 +940,7 @@ _start_scrub() { fi } +# Prepare everything and perform checks +_pre_work "$1" # Start SnapRAID Job -_start_job +_start From fd7acf704a608b319da139180e69c69d5bbba5ad Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Mon, 3 Jun 2024 00:06:46 -0500 Subject: [PATCH 10/15] remove OMV_MAIL_sender OMV_MAIL_sender was never set and never used, it might be an old option? --- usr/sbin/omv-snapraid-diff | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 5ceec6e..de57f42 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -631,7 +631,6 @@ _pre_work() { _log "DEBUG: PREHASH: $PREHASH" _log "DEBUG: SEND_SNAPRAID_CMD_LOGS: $SEND_SNAPRAID_CMD_LOGS" _log "DEBUG: ALWAYS_SEND_NOTIFICATION: $ALWAYS_SEND_NOTIFICATION" - _log "DEBUG: OMV_MAIL_sender: $OMV_MAIL_sender" _log "INFO: ----------------------------------------" fi @@ -687,7 +686,7 @@ _check_disks() { echo "Then, use 'snapraid check' to verify the integrity of the recovered files." >>$TMP_OUTPUT echo "Lastly, use 'snapraid sync' to update the parity files." >>$TMP_OUTPUT - _send_notification "ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "ERROR: Data disk not found!" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" NOTIFICATION_RETURN_CODE=$? _check_success $NOTIFICATION_RETURN_CODE && _rmtmp _log "WARN: script exits here due to missing disk ..." @@ -797,17 +796,17 @@ _start_diff() { # If update threshold exceeded elif [ $UPD_DEL_CHECK -eq 1 ]; then - _send_notification "WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "WARNING - Number of updated files (${UPDATE_COUNT}) exceeded threshold (${UPD_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" NOTIFICATION_RETURN_CODE=$? # If delete threshold exceeded elif [ $UPD_DEL_CHECK -eq 2 ]; then - _send_notification "WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "WARNING - Number of deleted files (${DEL_COUNT}) exceeded threshold (${DEL_THRESHOLD})" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" NOTIFICATION_RETURN_CODE=$? # IF both thresholds exceeded (UPD_DEL_CHECK -eq 3) else - _send_notification "WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "WARNING - Number of updated (${UPDATE_COUNT}) and deleted (${DEL_COUNT}) files exceeded updated (${UPD_THRESHOLD}) and deleted (${DEL_THRESHOLD}) thresholds" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" NOTIFICATION_RETURN_CODE=$? fi _check_success $NOTIFICATION_RETURN_CODE && _rmtmp @@ -816,7 +815,7 @@ _start_diff() { _log "INFO: No change detected. Nothing to do" if [ "$ALWAYS_SEND_NOTIFICATION" = "true" ]; then - _send_notification "No changes on data disks" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "No changes on data disks" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" fi fi @@ -875,7 +874,7 @@ _start_sync() { fi # Send job finished email - _send_notification "$NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" "${OMV_MAIL_sender}" + _send_notification "$NOTIFICATION_TITLE" "${TMP_OUTPUT}" "${NOTIFICATION_SUBJECT_PREFIX}" } _start_scrub() { From 1992f2ce103b08ba4f947eabd0937d6c6d8b47ac Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Mon, 3 Jun 2024 00:11:58 -0500 Subject: [PATCH 11/15] detect version at start --- usr/sbin/omv-snapraid-diff | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index de57f42..9fab2f2 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -54,6 +54,7 @@ TMP_OUTPUT="$TMP_DIR/snapraid.out" TMP_SCRUB_DIR="/root/.snapraid" SNAPRAID_BIN="/usr/bin/snapraid" +SNAPRAIDVERSION="$(dpkg-query -W -f='${Version}' snapraid)" ################################################################ ##### Functions @@ -489,8 +490,6 @@ _pre_work() { exit 1 fi - SNAPRAIDVERSION="$(dpkg-query -W -f='${Version}' snapraid)" - # check tmp-dirs if [ ! -d $TMP_DIR ]; then mkdir $TMP_DIR From 86e5362efdda00ee25e8cbd486a1878ba84ac47d Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Mon, 3 Jun 2024 00:27:08 -0500 Subject: [PATCH 12/15] add migration script --- .../conf.service.snapraid_7.0.11.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 usr/share/openmediavault/confdb/migrations.d/conf.service.snapraid_7.0.11.sh diff --git a/usr/share/openmediavault/confdb/migrations.d/conf.service.snapraid_7.0.11.sh b/usr/share/openmediavault/confdb/migrations.d/conf.service.snapraid_7.0.11.sh new file mode 100644 index 0000000..c6ab708 --- /dev/null +++ b/usr/share/openmediavault/confdb/migrations.d/conf.service.snapraid_7.0.11.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +set -e + +. /usr/share/openmediavault/scripts/helper-functions + +SERVICE_XPATH="/config/services/snapraid" + +if ! omv_config_exists "${SERVICE_XPATH}/apprise"; then + omv_config_add_key "${SERVICE_XPATH}" "apprise" "0" +fi + +if ! omv_config_exists "${SERVICE_XPATH}/apprisecmd"; then + omv_config_add_key "${SERVICE_XPATH}" "apprisecmd" "" +fi + +if ! omv_config_exists "${SERVICE_XPATH}/ntfy"; then + omv_config_add_key "${SERVICE_XPATH}" "ntfy" "0" +fi + +if ! omv_config_exists "${SERVICE_XPATH}/ntfycmd"; then + omv_config_add_key "${SERVICE_XPATH}" "ntfycmd" "" +fi + +if ! omv_config_exists "${SERVICE_XPATH}/sendsnaplog"; then + omv_config_add_key "${SERVICE_XPATH}" "sendsnaplog" "0" +fi + +if ! omv_config_exists "${SERVICE_XPATH}/sendnotifalways"; then + omv_config_add_key "${SERVICE_XPATH}" "sendnotifalways" "0" +fi + +exit 0 From 1c61524f9a17ee178457ff8a5d421fbedc478531 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Mon, 3 Jun 2024 00:30:57 -0500 Subject: [PATCH 13/15] bump script version --- usr/sbin/omv-snapraid-diff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 9fab2f2..90da577 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -32,7 +32,7 @@ # Changes from v 1.1.0: Fix deleted and updated files logic # Changes from v 1.2.0: Add support for multiple arrays -SCRIPTVERSION="1.3.5" +SCRIPTVERSION="1.3.6" # get basic info . /etc/default/openmediavault From 1c5ce0b16f6eb24e329d1e09b195c1fc9a56a7c4 Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Mon, 3 Jun 2024 02:52:32 -0500 Subject: [PATCH 14/15] round max days, fix SnapRAID SCRUB-Cycle count comparison --- usr/sbin/omv-snapraid-diff | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index 90da577..d20e3fd 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -392,6 +392,17 @@ seconds_to_days() { echo $(($1 / 60 / 60 / 24)) } +_calc_max_days() { + local QUOT=$((100 * $1 / $2)) + local REMA=$((100 * $1 % $2)) + + if [ $REMA -eq 0 ]; then + echo $QUOT + else + echo $((QUOT + 1)) + fi +} + ################################################################ ##### MAIN BODY of the script ################################################################ @@ -881,6 +892,7 @@ _start_scrub() { local CURRENT_DATE_DATE local DAYS_DIFF_DATE local DAYS_DIFF_EPOCH + local MAX_NUM_DAYS SCRUB_SUCCESS="no" @@ -890,13 +902,13 @@ _start_scrub() { DAYS_DIFF_EPOCH=$((CURRENT_DATE_EPOCH - $(date_to_epoch $LAST_SCRUB_DATE))) DAYS_DIFF_DATE=$(seconds_to_days $DAYS_DIFF_EPOCH) - if [ $LAST_SCRUB_DATE -ne 0 ] && [ $DAYS_DIFF -lt $(($(date_to_seconds $SCRUB_FREQUENCY_IN_DAYS))) ]; then + if [ $LAST_SCRUB_DATE -ne 0 ] && [ $DAYS_DIFF_EPOCH -lt $(($(date_to_seconds $SCRUB_FREQUENCY_IN_DAYS))) ]; then _log "INFO: SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run." echo "SnapRAID SCRUB-Cycle count ($SCRUB_FREQUENCY_IN_DAYS) not met ($DAYS_DIFF_DATE). No scrub was run. - $(date)" >>$TMP_OUTPUT SCRUB_SUCCESS="none" else if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ne 0 ]; then - local MAX_NUM_DAYS=$((100 * SCRUB_FREQUENCY_IN_DAYS / SCRUB_PERCENT)) + MAX_NUM_DAYS=$(_calc_max_days "$SCRUB_FREQUENCY_IN_DAYS" "$SCRUB_PERCENT") if [ $ACCUMULATED_DAYS_SCRUBBED_FILES -ge $MAX_NUM_DAYS ]; then # We reached the max num of days, this means we must restart and scrub everything again, # in an ideal (non-file-changing) scenario we scrubbed all the data From f5ae95ed97c87f54b074b8af9d267c6ecceb54fd Mon Sep 17 00:00:00 2001 From: Angel <1024mb@users.noreply.github.com> Date: Mon, 3 Jun 2024 03:04:13 -0500 Subject: [PATCH 15/15] delete tar file after sending --- usr/sbin/omv-snapraid-diff | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usr/sbin/omv-snapraid-diff b/usr/sbin/omv-snapraid-diff index d20e3fd..5eb33fa 100755 --- a/usr/sbin/omv-snapraid-diff +++ b/usr/sbin/omv-snapraid-diff @@ -116,6 +116,10 @@ EOF fi fi + if [ -f "$SNAP_LOG_ARCHIVE_PATH" ]; then + rm -f "$SNAP_LOG_ARCHIVE_PATH" + fi + if [ "$MAIL_FAIL" = "true" ] || [ "$APPRISE_FAIL" = "true" ] || [ "$NTFY_FAIL" = "true" ]; then return 1 else