Refactored scan scripts so they work with non standard directories

.dockerignore

@@ -17,6 +17,7 @@
 !cmake/
 !data/
 !hclasses/
+!scripts/
 !python/
 !tests/AdagucTests/
 !tests/expectedoutputs/

Docker/adaguc-server-addfile.sh

@@ -1,90 +1,2 @@
 #!/bin/bash
-
-# TODO: https://github.com/KNMI/adaguc-server/issues/71
-
-# Author: [email protected]
-# 2020-04-17
-#
-# This script is for scanning files with adaguc-server. Adaguc can detect automatically to which dataset(s) the file belongs.
-#
-# With -f you can optionally specify the file to add
-# With -d you can optionally specify the dataset to add.
-# If only a file is specified without a dataset, adaguc-server will try to find the matching dataset that belongs to the file.
-#
-# Usage: ./adaguc-server-addfile.sh -d <datasetname (optional)> -f <file to add>
-# Usage: ./adaguc-server-addfile.sh -f <file to add> 
-# Usage: ./adaguc-server-addfile.sh -d <datasetname> 
-
-
-THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
-
-ADAGUC_DATASET=''
-
-while getopts "d:f:" o; do
-    case "${o}" in
-        d)
-            ADAGUC_DATASET=${OPTARG}
-            ;;
-        f)
-            ADAGUC_DATAFILE=${OPTARG}
-            ;;
-        h)
-            usage
-            ;;
-        *)
-            usage
-            ;;
-    esac
-done
-
-
-### Scan a file for a specified dataset ###
-
-
-if [[ -n "${ADAGUC_DATASET}" &&  -n "${ADAGUC_DATAFILE}" ]]; then
-  STATUSCODE=0
-  echo "Adding file [${ADAGUC_DATAFILE}] to dataset [${ADAGUC_DATASET}]:"
-  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${ADAGUC_DATASET} --path ${ADAGUC_DATAFILE}"
-  echo $command
-  $command
-  OUT=$?
-  if [ ${OUT} -ne 0 ]; then
-    STATUSCODE=1
-  fi
-  exit ${STATUSCODE} 
-fi
-
-
-### Scan a file ###
-if [[ -n "${ADAGUC_DATAFILE}" ]]; then
-
-  basediroffile=${ADAGUC_DATAFILE%/*}
-  basenameoffile="${ADAGUC_DATAFILE##*/}"
-
-  STATUSCODE=0
-  echo "Adding file [${ADAGUC_DATAFILE}] to dataset [${alldatasets}]:"
-  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --autofinddataset --verboseoff --config ${ADAGUC_CONFIG} --path ${ADAGUC_DATAFILE}"
-  echo $command
-  $command
-  OUT=$?
-  if [ ${OUT} -ne 0 ]; then
-    STATUSCODE=1
-  fi
-  exit ${STATUSCODE} 
-fi
-
-# Scan a dataset
-if [[ -n "${ADAGUC_DATASET}" ]]; then
-  STATUSCODE=0
-  echo "Scanning full dataset [${ADAGUC_DATASET}]:"
-  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --verboseoff --config ${ADAGUC_CONFIG},${ADAGUC_DATASET}"
-  echo $command
-  $command
-  OUT=$?
-  if [ ${OUT} -ne 0 ]; then
-    STATUSCODE=1
-  fi
-  exit ${STATUSCODE} 
-fi
+. ${ADAGUC_PATH}/scripts/scan.sh

Docker/adaguc-server-updatedatasets.sh

@@ -1,61 +1,7 @@
 #!/bin/bash
-
-# nullglob is a Bash shell option which modifies [[glob]] expansion such
-# that patterns that match no files expand to zero arguments, rather than to themselves.
 shopt -s nullglob
-
-# This script requires ADAGUC_CONFIG to be set. In case of running locally with adaguc-services, this is probably
-# `export ADAGUC_CONFIG=./data/config/adaguc.vm.xml`
-
-# TODO: https://github.com/KNMI/adaguc-server/issues/71
-
-THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
-
-# Unbufferd logging for realtime output
-export ADAGUC_ENABLELOGBUFFER=FALSE
-
-
-# Do export ADAGUC_VERBOSE="--verboseon" to enable verbose logging
-
-ADAGUC_VERBOSE="${ADAGUC_VERBOSE:=--verboseoff}" 
-
 if [[ $1 ]]; then
-  # Update a specific dataset
-  for configfile in /data/adaguc-datasets/$1 ;do
-    filename=/data/adaguc-datasets/"${configfile##*/}" 
-    filebasename=${filename##*/}
-    echo ""
-    if [[ $2 ]]; then
-      echo "*** Starting update with tailpath $2 for dataset ${filename}" 
-      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} --tailpath $2
-      OUT=$?
-    else
-      echo "*** Starting update for ${filename}" 
-      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
-      OUT=$?
-    fi
-
-    echo ""
-  done
-
+  . ${ADAGUC_PATH}/scripts/scan.sh -d $1
 else
-
-  # Update all datasets
-  for configfile in /data/adaguc-datasets/*xml ;do
-    filename=/data/adaguc-datasets/"${configfile##*/}" 
-    filebasename=${filename##*/}
-    if [[ "${ADAGUC_DATASET_MASK}" && `echo ${filebasename} | grep -E ${ADAGUC_DATASET_MASK}` != ${filebasename} ]] ; then
-        if [[ "${ADAGUC_DATASET_MASK}" ]] ; then
-            echo "${filebasename} doesn't match ${ADAGUC_DATASET_MASK}"
-        fi
-        continue
-    fi
-    echo ""
-    echo "Starting update for ${filename}" 
-    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
-    OUT=$?
-     done
-
+. ${ADAGUC_PATH}/scripts/scan.sh -d "*"
 fi

Dockerfile

@@ -123,13 +123,15 @@ COPY ./Docker/adaguc-server-config-python-postgres.xml /adaguc/adaguc-server-con
 COPY ./Docker/start.sh /adaguc/
 COPY ./Docker/adaguc-server-*.sh /adaguc/
 COPY ./Docker/baselayers.xml /adaguc/adaguc-datasets-internal/baselayers.xml
+COPY scripts /adaguc/adaguc-server-master/scripts
+COPY ./scripts/*.sh /adaguc/
 # Copy pgbouncer and supervisord config files
 COPY ./Docker/pgbouncer/ /adaguc/pgbouncer/
 COPY ./Docker/supervisord/ /etc/supervisor/conf.d/
 COPY ./Docker/run_supervisord.sh /adaguc/run_supervisord.sh
 # Set permissions
-RUN  chmod +x /adaguc/adaguc-server-*.sh && \
-    chmod +x /adaguc/start.sh && \
+RUN  chmod +x /adaguc/*.sh && \
+    chmod +x /adaguc/adaguc-server-master/scripts/*.sh && \
     chown -R adaguc:adaguc /data/adaguc* /adaguc /adaguc/*
 
 ENV ADAGUC_PATH=/adaguc/adaguc-server-master