From 815680c0e5c51c63401ebe5d512cc77b44c12219 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Thu, 9 Aug 2018 14:27:22 +0200 Subject: [PATCH 1/2] renameFastQs.bash: exit 1 when one or more FastQs failed QC. --- renameFastQs.bash | 50 ++++++++++++++++++----------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/renameFastQs.bash b/renameFastQs.bash index 777f1c6..41a4331 100755 --- a/renameFastQs.bash +++ b/renameFastQs.bash @@ -88,29 +88,9 @@ function _RenameFastQ() { local _fastqFile="$(basename "${_fastqPath}")" # - # Do NOT parse filenames: get the essential stuff from the sequence read IDs instead + # Get essential meta-data from the sequence read IDs in the FastQ file. + # (Do NOT rely on parsing FastQ filenames!) # - #local _regex='^([A-Z0-9]*XX)_(103373-[0-9][0-9]*-[0-9][0-9]*)_([ATCG][ATCG]*-[ATCG][ATCG]*)_L00([1-8])_R([12]).fastq.gz$' - #if [[ "${_fastqFile}" =~ ${_regex} ]] - #then - # local _flowcell="${BASH_REMATCH[1]}" - # local _customerBatchSampleCombi="${BASH_REMATCH[2]}" - # local _barcodes="${BASH_REMATCH[3]}" - # local _lane="${BASH_REMATCH[4]}" - # local _sequenceReadOfPair="${BASH_REMATCH[5]}" - # if [[ "${enableVerboseLogging}" -eq 1 ]] - # then - # echo "DEBUG: Found _flowcell ............... = ${_flowcell}" - # echo "DEBUG: Found _customerBatchSampleCombi = ${_customerBatchSampleCombi}" - # echo "DEBUG: Found _barcodes ............... = ${_barcodes}" - # echo "DEBUG: Found _lane ................... = ${_lane}" - # echo "DEBUG: Found _sequenceReadOfPair ..... = ${_sequenceReadOfPair}" - # fi - #else - # echo "FATAL: Failed to parse filename for ${_fastq}" - # exit 1 - #fi - local _firstReadID=$(zcat "${_fastqPath}" | head -1) if [[ "${enableVerboseLogging}" -eq 1 ]] then @@ -177,13 +157,15 @@ function _RenameFastQ() { fi elif [[ "${_mostAbundandBarcode}" =~ N ]] then - echo "ERROR: Most abundant barcode(s) in max 1000 reads from middle of FastQ contains Ns: ${_barcodes}" + qualityControl='failed' + echo "ERROR: Most abundant barcode(s) in max 1000 reads from middle of FastQ contains Ns: ${_mostAbundandBarcode}." echo "ERROR: Skipping discarded FastQ ${_fastqFile} due to poor sequencing quality of barcode(s)." return else + qualityControl='failed' echo "ERROR: Failed to determine the most abundant barcode(s) from max 1000 reads from middle of FastQ." - echo "FATAL: Failed to process FastQ ${_fastqFile} due to poor sequencing quality of barcode(s)." - exit 1 + echo "ERROR: Failed to parse barcodes from read IDs of FastQ file ${_fastqFile}." + return fi local _fastqChecksum=$(cat "${_fastqDir}/"*.md5 | grep "${_fastqFile}" | awk '{print $1}') @@ -225,7 +207,7 @@ function _RenameFastQ() { enableVerboseLogging=0 # Disabled by default. while getopts "s:f:hv" opt do - case $opt in + case ${opt} in h) _Usage # @@ -277,20 +259,26 @@ if [[ "${sequencingStartDate}" =~ ${ssd_regex} ]] then echo "INFO: Using sequencingStartDate ${sequencingStartDate}" else - _reportError ${LINENO} 1 "sequencingStartDate in unsupported format. Must be YYMMDD, but got ${sequencingStartDate}." + _reportError ${LINENO} 1 "sequencingStartDate in unsupported format. Must be YYMMDD, but got ${sequencingStartDate}." fi # # Process FastQ files. # +qualityControl='unknown' for FastQ in $(ls -1 ${fastqFilePattern}) do _RenameFastQ "${FastQ}" "${sequencingStartDate}" done # -# Reset trap and exit. +# Reset trap and exit with 0 or 1 depending on wether QC was Ok or failed. # -echo "INFO: Finished successfully!" -trap - EXIT -exit 0 \ No newline at end of file +if [[ "${qualityControl}" == 'failed' ]] +then + _reportError ${LINENO} 1 "One or more FastQ files failed QC and was not renamed!." +else + echo "INFO: Finished successfully!" + trap - EXIT + exit 0 +fi \ No newline at end of file From 3e27c99bd2fd158b425c6d09eda46ec82b0e0063 Mon Sep 17 00:00:00 2001 From: pneerincx Date: Thu, 9 Aug 2018 17:12:31 +0200 Subject: [PATCH 2/2] Fixed bugs in error handling and logging. --- renameFastQs.bash | 53 +++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/renameFastQs.bash b/renameFastQs.bash index 24a5ba2..2373776 100755 --- a/renameFastQs.bash +++ b/renameFastQs.bash @@ -46,7 +46,7 @@ function _Usage() { # # Custom signal trapping functions (one for each signal) required to format log lines depending on signal. # -function trapSig() { +function _trapSig() { local _trap_function="${1}" local _line="${2}" local _function="${3}" @@ -57,22 +57,39 @@ function trapSig() { done } -function trapHandler() { +function _trapHandler() { local _signal="${1}" local _problematicLine="${2}" local _function="${3}" local _exitStatus="${4}" + local _errorMessage="Unknown error in function ${_function}." + _reportFatalError "${_problematicLine}" "${_exitStatus}" "${_errorMessage}" +} + +function _reportFatalError() { + local _problematicLine="${1}" + local _exitStatus="${2:-$?}" local _errorMessage="Unknown error." + _errorMessage="${3:-${_errorMessage}}" + # + # Notify on STDOUT. + # echo " - $(hostname) - ${SCRIPT_NAME}:${_problematicLine}: FATAL: exit code = ${_exitStatus} - $(hostname) - ${SCRIPT_NAME}:${_problematicLine}: error message = ${_errorMessage} +$(hostname) - ${SCRIPT_NAME}:${_problematicLine}: FATAL: exit code = ${_exitStatus} +$(hostname) - ${SCRIPT_NAME}:${_problematicLine}: error message = ${_errorMessage} " + # + # Reset trap and exit. + # + trap - EXIT + exit 1 } + # # Trap all exit signals: HUP(1), INT(2), QUIT(3), TERM(15), ERR # -trapSig 'trapHandler' '${LINENO}' '${FUNCNAME:-main}' '$?' HUP INT QUIT TERM EXIT ERR +_trapSig '_trapHandler' '${LINENO}' '${FUNCNAME:-main}' '$?' HUP INT QUIT TERM EXIT ERR function _RenameFastQ() { local _fastqPath="${1}" @@ -121,8 +138,7 @@ function _RenameFastQ() { # if [[ "${#_run}" -lt 3 ]] then - echo "FATAL: run number detected in ID of first read is too short (< 3): ${_run}" - exit 1 + _reportFatalError ${LINENO} '1' 'Run number detected in ID of first read is too short (< 3): '"${_run}." elif [[ "${#_run}" -eq 3 ]] then _run="0${_run}" @@ -136,8 +152,7 @@ function _RenameFastQ() { echo "DEBUG: Found _sequenceReadOfPair ..... = ${_sequenceReadOfPair}" fi else - echo "FATAL: Failed to parse required meta-data values from ID of first read of ${_fastqPath}" - exit 1 + _reportFatalError ${LINENO} '1' "Failed to parse required meta-data values from ID of first read of ${_fastqPath}". fi # @@ -170,8 +185,8 @@ function _RenameFastQ() { return else qualityControl='failed' - echo "ERROR: Failed to determine the most abundant barcode(s) from max 1000 reads from middle of FastQ." - echo "ERROR: Failed to parse barcodes from read IDs of FastQ file ${_fastqFile}." + echo "ERROR: Failed to determine the most abundant barcodes from max 1000 reads from middle of FastQ." + echo "ERROR: Failed to parse barcode(s) from read IDs of FastQ file ${_fastqFile}." return fi @@ -189,9 +204,7 @@ function _RenameFastQ() { # if [[ -e "${_newFastqDir}/${_newFastqFile}" ]] then - echo "FATAL: ${_newFastqDir}/${_newFastqFile} already exists." - echo "FATAL: will NOT move ${_fastqPath} -> ${_newFastqDir}/${_newFastqFile}." - exit 1 + _reportFatalError ${LINENO} '1' "${_newFastqDir}/${_newFastqFile} already exists; will NOT move ${_fastqPath} -> ${_newFastqDir}/${_newFastqFile}." fi # @@ -233,10 +246,10 @@ do enableVerboseLogging=1 ;; \?) - _reportError ${LINENO} '1' "Invalid option -${OPTARG}. Try \"$(basename $0) -h\" for help." + _reportFatalError "${LINENO}" '1' "Invalid option -${OPTARG}. Try \"$(basename $0) -h\" for help." ;; :) - _reportError ${LINENO} '1' "Option -${OPTARG} requires an argument. Try \"$(basename $0) -h\" for help." + _reportFatalError "${LINENO}" '1' "Option -${OPTARG} requires an argument. Try \"$(basename $0) -h\" for help." ;; esac done @@ -247,7 +260,7 @@ done shift $(($OPTIND - 1)) if [[ ! -z ${1:-} ]] then - _reportError ${LINENO} '1' "Invalid argument \"$1\". Try \"$(basename $0) -h\" for help." + _reportFatalError "${LINENO}" '1' "Invalid argument \"$1\". Try \"$(basename $0) -h\" for help." fi # @@ -255,7 +268,7 @@ fi # if [[ -z "${sequencingStartDate:-}" || -z "${fastqFilePattern:-}" ]] then - _reportError ${LINENO} 1 "One ore more required arguments is missing. Try \"$(basename $0) -h\" for help." + _reportFatalError "${LINENO}" '1' "One ore more required arguments is missing. Try \"$(basename $0) -h\" for help." fi # @@ -266,7 +279,7 @@ if [[ "${sequencingStartDate}" =~ ${ssd_regex} ]] then echo "INFO: Using sequencingStartDate ${sequencingStartDate}" else - _reportError ${LINENO} 1 "sequencingStartDate in unsupported format. Must be YYMMDD, but got ${sequencingStartDate}." + _reportFatalError "${LINENO}" '1' "sequencingStartDate in unsupported format. Must be YYMMDD, but got ${sequencingStartDate}." fi # @@ -283,7 +296,7 @@ done # if [[ "${qualityControl}" == 'failed' ]] then - _reportError ${LINENO} 1 "One or more FastQ files failed QC and was not renamed!." + _reportFatalError "${LINENO}" '1' "One or more FastQ files failed QC and was not renamed!." else echo "INFO: Finished successfully!" trap - EXIT