dset

#!/bin/bash
source ~/.shcolor.sh 2>/dev/null || source <(curl -s https://raw.githubusercontent.com/kba/shcolor/master/shcolor.sh|tee ~/.shcolor.sh)

loginfo () { echo -e "[\e[32minfo\e[0m] $(date +"%T.%N") $*" >&2; }

#
# Configuration
#

# Root directory of the repo, default the directory of this script
DIR_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# Set this to the Infolink JAR
# INFOLINK_JAR=$DIR_ROOT/infoLink/build/libs/infoLink-1.0.jar
INFOLINK_BIN=$DIR_ROOT/infoLink/build/distributions/infoLink-1.0/bin/infoLink
# Datasets subdirectory
DIR_DATASET="$DIR_ROOT/datasets"
# Template subdirectotry
DIR_TEMPLATE="$DIR_ROOT/template"
# Frontend API Base URL
FRONTEND_BASEURL="http://infolis.gesis.org/infolink/api"
# Arguments to the CLI call
declare -A JAVA_CLI_ARGS
# Boolean arguments to the CLI call
JAVA_CLI_FLAGS=()

if [[ -e "$DIR_ROOT/dset.conf" ]];then
    source "$DIR_ROOT/dset.conf"
fi

OPT_DEBUG=true
OPT_FORCE=false
OPT_TO_TEXT=true
OPT_TIMESTAMP=true

#
# Usage
#

usage() {
    if [[ ! -z "$1" ]];then
        echo "`C 1`ERROR:`C` $1"
    fi
    echo ""
    echo "  `C 15`$0 `C 2`<cmd>`C` [`C 12`options`C`] [`C 11`<dataset> <subset> <json> <queryfile>`C`]"
    echo ""
    echo "  Options:"
    echo ""
    echo "    `C 12`--force`C`              Overwrite if exists"
    echo "    `C 12`--debug -d`C`           Output debug information"
    echo "    `C 12`--convert-to-text`C`    Convert the data to text before execution"
    echo "    `C 12`--no-convert-to-text`C` Do NOT convert the data to text before execution"
    echo "    `C 12`--timestamp`C`          Prefix uploaded resources with a timestamp"
    echo "    `C 12`--no-timestamp`C`       Do NOT Prefix uploaded resources with a timestamp"
    echo "    `C 12`--tag <tag>`C`          Override tag to be used (default: `C 11`<DATASET>-<SUBSET>-<JSON>-<TIMESTAMP>`C`"
    echo ""
    echo "  Subcommands:"
    echo ""
    echo "  `C 2`init`C 11` <dataset>`C`"
    echo "    Initialize new dataset in directory datasets/<dataset>"
    echo "  `C 2`pre-commit`C 11` <dataset>`C`"
    echo "    Validate conformity before git-commit"
    echo "  `C 2`checksum`C` [`C 12`--force`C`] `C 11`<dataset>`C`"
    echo "    Generate checksum for dataset <dataset>"
    echo "  `C 2`to-text`C 11` <dataset> <subset>`C`"
    echo "    Convert the *.pdf in subset `C 11`<subset>`C` of dataset `C 11`<dataset>`C` to text/plain"
    echo "  `C 2`exec`C 11` <dataset> <subset> <json>`C`"
    echo "    Execute an algorithm on data from subset `C 11`<subset>`C` of dataset `C 11`<dataset>`C` using execution `C 11`<json>`C`"
    echo "  `C 2`query-for-subset`C 11` <dataset> <queryfile>`C`"
    echo "    Search dataset `C 11`<dataset>`C` for documents matching any line in `C 11`<queryfile>`C`"
    echo "  `C 2`upload`C 11` <dataset> <tag>`C`"
    echo "    Upload the database `C 11`<tag>`C` in dataset `C 11`<dataset>`C` to the server"
    echo "  `C 2`text-to-tsv`C 11` <dataset> <subset>`C`"
    echo "    Convert the `C 11`<subset>`C` in dataset `C 11`<dataset>`C` to WebAnno-compatible TSV"
    exit
}

#
# Helpers
#

die_if_not_exists() {
    actual_file=''
    if [[ ! -z "$2" ]];then
        actual_file="(=$2)"
    fi
    if [[ ! -e "$1" ]];then
        echo "$1 $actual_file does not exist!"
        exit 1
    fi
}
die_if_exists() {
    if [[ -e "$1" ]];then
        echo "$1 already exists!"
        exit 1
    fi
}
die_if_empty() {
    if [[ -z "$1" ]];then
        echo "Argument $2 must not be empty!"
        exit 1;
    fi
}

cd_root() {
    cd $DIR_ROOT
}

cd_dataset() {
    die_if_not_exists "$DIR_DATASET/$DATASET"
    cd "$DIR_DATASET/$DATASET"
}

default_tag() {
    echo "$DATASET-$SUBSET-$JSON-$(date "+%Y-%m-%dT%H-%M-%S")"
}

exec_java() {
    echo "`C 11`About to execute:`C`"
    # echo "$INFOLINK_BIN ${JAVA_CLI_ARGS[@]} "
    # echo "$JAVA_HOME/bin/java -jar $INFOLINK_JAR"
    local args=()
    JAVA_CLI_ARGS["--meta-dir"]="$DIR_DATASET/$DATASET/meta"
    if [[ ! -z "$TAG" ]];then
        echo "--tag $TAG"
        JAVA_CLI_ARGS["--tag"]="$TAG"
    fi
    for arg in "${!JAVA_CLI_ARGS[@]}";do
        echo "$arg ${JAVA_CLI_ARGS[$arg]}"
        args+=("$arg" "${JAVA_CLI_ARGS[$arg]}")
    done
    for flag in "${JAVA_CLI_FLAGS[@]}";do
        echo "$flag"
        args+=("$flag")
    done
    echo "$INFOLINK_BIN ${args[*]}"
    echo "<ENTER> to continue, <CTRL-C> to quit"
    read
    $INFOLINK_BIN "${args[@]}"
}


#
# Commands
#

cmd_sha256() {
    cd_dataset
    [[ $OPT_FORCE == true ]] || die_if_exists SHA256SUMS
    echo "Creating SHA256SUMS ..."
    echo "" > SHA256SUMS
    find . \
        -type f \
        -regex './\(pdf\|meta\|txt\).*' \
        | xargs \
            -P 10 \
            -n 5 \
            -I{} \
            sha256sum {} >> SHA256SUMS
    cd_root
}

cmd_init() {
    die_if_empty "$DATASET" "<dataset>"
    die_if_exists "$DIR_DATASET/$DATASET" 
    cp -rv $DIR_TEMPLATE "$DIR_DATASET/$DATASET"
    cd_dataset
    sed -i "s/TITLE/$DATASET/" README.md
    cd_root
}

cmd_to_text() {
    die_if_empty "$DATASET" "<dataset>"
    die_if_empty "$SUBSET" "<subset>"
    die_if_not_exists "$DIR_DATASET/$DATASET/subsets/$SUBSET"

    cd_dataset

    JAVA_CLI_ARGS["--pdf-dir"]="$DIR_DATASET/$DATASET/subsets/$SUBSET"
    JAVA_CLI_ARGS["--text-dir"]="$DIR_DATASET/$DATASET/text/$SUBSET"
    JAVA_CLI_ARGS["--db-dir"]="$PWD/db"
    JAVA_CLI_ARGS["--tag"]="to-text"
    exec_java

    cd_root
}

cmd_query_subset() {
    die_if_empty "$DATASET" "<dataset>"
    die_if_empty "$QUERYFILE" "<queryfile>"
    qFile="$DIR_DATASET/$DATASET/$QUERYFILE.lst"
    indexDir="$DIR_DATASET/$DATASET/index"
    dbDir="$DIR_DATASET/$DATASET/db"
    die_if_not_exists "$qFile"
    mkdir -p $indexDir

    cd_dataset

    JAVA_CLI_FLAGS+=("--search-candidates")
    JAVA_CLI_ARGS["--pdf-dir"]="$DIR_DATASET/$DATASET/subsets/all"
    JAVA_CLI_ARGS["--text-dir"]="$DIR_DATASET/$DATASET/text/all"
    JAVA_CLI_ARGS["--db-dir"]="$dbDir"
    JAVA_CLI_ARGS["--tag"]="subset-$QUERYFILE"
    JAVA_CLI_ARGS["--queries-file"]="$qFile"
    JAVA_CLI_ARGS["--index-dir"]="$indexDir"

    exec_java

    mkdir "subsets/$QUERYFILE"
    while read fname; do
        ln -s "../$fname.pdf" "subsets/$QUERYFILE"
    done < "$DIR_DATASET/$DATASET/db/$QUERYFILE"

    cd_root
}

cmd_text_to_tsv() {
    die_if_empty "$DATASET" "<dataset>"
    die_if_empty "$SUBSET" "<subset>"
    mkdir -p "datasets/$DATASET/tsv/$SUBSET"
    cd_dataset
    cd_root
    python txt2tsv.py "datasets/$DATASET/text/$SUBSET" "datasets/$DATASET/tsv/$SUBSET"
}

cmd_exec() {
    die_if_empty "$DATASET" "<dataset>"
    die_if_empty "$SUBSET" "<subset>"
    die_if_empty "$JSON" "<json>"
    die_if_empty "$TAG" "<tag>"
    cd_dataset
    die_if_not_exists "$JSON.json" "$PWD/$JSON.json"
    die_if_not_exists "$PWD/subsets/$SUBSET"

    JAVA_CLI_ARGS["--json"]="$PWD/$JSON.json"
    JAVA_CLI_ARGS["--pdf-dir"]="$PWD/subsets/$SUBSET"
    JAVA_CLI_ARGS["--db-dir"]="$PWD/db"
    JAVA_CLI_ARGS["--text-dir"]="$PWD/text/$SUBSET"
    exec_java

    cd_root
}

cmd_upload() {
    die_if_empty "$DATASET" "<dataset>"
    die_if_empty "$TAG" "<tag>"

    dbDump="$DIR_DATASET/$DATASET/db/$TAG.json"
    die_if_not_exists "$dbDump"

    cd_dataset

    # Map UUID -> filename
    declare -A FILES
    declare -a fileNames
    loginfo "Grepping fileNames"
    local IFS=$'\n'
    fileNames=($(grep "fileName" "$dbDump"))
    fileNamesLen=${#fileNames[@]}
    loginfo "Grepped ${#fileNames[@]} fileNames"
    loginfo "Transform to Map<UUID,Filename>"
    i=0
    for line in "${fileNames[@]}";do
        uuid=${line#*\"}
        uuid=${uuid%%\"*}
        fileName=${line#*fileName\":\"}
        fileName=${fileName%%\"*}
        if (( i++ % 1000 == 1 ));then
            loginfo "[$i / ${#fileNames[@]}] $uuid -> ...${fileName##*/}";
        fi
        FILES["$uuid"]="$fileName"
    done
    loginfo "DoneTransform to Map<UUID,Filename>"

    url="$FRONTEND_BASEURL/json-import"
    curl_args=()
    curl_args+=("$url")
    if [[ "$OPT_TIMESTAMP" != true ]];then
        url+='?no-timestamp=no-timestamp'
    fi
    curl_args+=("-XPOST")
    curl_args+=("-F" "tags=$TAG")
    curl_args+=("-F" "db=<$dbDump")
    for uuid in "${!FILES[@]}";do
        curl_args+=("-F" "$uuid=@${FILES[$uuid]}")
    done

    curlfile=$(mktemp "/tmp/dset-upload.XXXXXXX.curl")
    loginfo "Start writing curl args to $curlfile"
    local IFS=$'\n'
    printf '%s\n' "${curl_args[@]}" > "$curlfile"
    loginfo "Finished writing curl args to $curlfile"
    # curl -s "${curl_args[@]}"

    cd_root
}

debug() {
    loginfo "`C 13`#DEBUG>   `C 2`CMD`C`: $CMD"
    loginfo "`C 13`#DEBUG>   `C 11`DATASET`C`: $DATASET"
    loginfo "`C 13`#DEBUG>   `C 11`SUBSET`C`: $SUBSET"
    loginfo "`C 13`#DEBUG>   `C 11`JSON`C`: $JSON"
    loginfo "`C 13`#DEBUG>   `C 12`OPT_FORCE`C`: $OPT_FORCE"
    loginfo "`C 13`#DEBUG>   `C 12`OPT_TO_TEXT`C`: $OPT_TO_TEXT"
    loginfo "`C 13`#DEBUG>   `C 12`OPT_TAG`C`: $OPT_TAG"
    loginfo "`C 13`#DEBUG>   `C 12`OPT_DEBUG`C`: $OPT_DEBUG"
    loginfo "`C 13`#DEBUG>   `C 14`\$@`C`: $@"
}

CMD="$1" && shift
[[ -z "$CMD" ]] && usage "Must specify cmd"

while [[ "$1" =~ ^- ]];do
    case "$1" in 
        --force) 
            OPT_FORCE=true
            ;;
        --tag)
            OPT_TAG=$2
            shift
            ;;
        --convert-to-text)
            OPT_TO_TEXT=true
            ;;
        --no-convert-to-text)
            OPT_TO_TEXT=false
            ;;
        --nodebug|-D)
            OPT_DEBUG=false
            ;;
        --timestamp)
            OPT_TIMESTAMP=true
            ;;
        --no-timestamp)
            OPT_TIMESTAMP=false
            ;;
        --debug|-d)
            OPT_DEBUG=true
            ;;
        *)
            usage "Unknown Option: '$1'"
            ;;
    esac
    shift;
done

if [[ "$OPT_TO_TEXT" == true ]];then
    JAVA_CLI_FLAGS+=("--convert-to-text")
fi

case "$CMD" in
    init)
        DATASET="$1" && shift
        [[ -z "$DATASET" ]] && usage "Must specify dataset"
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_init $@
        ;;
    checksum)
        DATASET="$1" && shift
        [[ -z "$DATASET" ]] && usage "Must specify dataset"
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_sha256 $@
        ;;
    exec)
        DATASET="$1" && shift
        SUBSET="$1" && shift
        JSON="$1" && shift
        TAG=${OPT_TAG:-$(default_tag)}
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_exec $@
        ;;
    query-for-subset)
        DATASET="$1" && shift
        QUERYFILE="$1" && shift
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_query_subset $@
        ;;
    to-text)
        DATASET="$1" && shift
        SUBSET="$1" && shift
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_to_text $@
        ;;
    text-to-tsv)
        DATASET="$1" && shift
        SUBSET="$1" && shift
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_text_to_tsv $@
        ;;
    upload)
        DATASET="$1" && shift
        TAG=$OPT_TAG && shift
        [[ $OPT_DEBUG == true ]] && debug $@
        cmd_upload $@
        ;;
    *)
        usage "Unknown command '$CMD'"
        ;;
esac