From 322cb973daa85a93ecba4342c0e0b2964288ac9a Mon Sep 17 00:00:00 2001
From: Maarten Plieger <maarten.plieger@knmi.nl>
Date: Thu, 17 Oct 2024 16:11:33 +0200
Subject: [PATCH 1/4] Refactored scan scripts so they work with non standard
 directories

---
 Docker/Readme.md                       |   1 -
 Docker/adaguc-server-addfile.sh        |  89 +--------------------
 Docker/adaguc-server-chkconfig.sh      |  20 ++++-
 Docker/adaguc-server-updatedatasets.sh |  45 +----------
 Docker/scan.sh                         | 106 +++++++++++++++++++++++++
 5 files changed, 127 insertions(+), 134 deletions(-)
 delete mode 100644 Docker/Readme.md
 create mode 100755 Docker/scan.sh

diff --git a/Docker/Readme.md b/Docker/Readme.md
deleted file mode 100644
index 7a08011d0..000000000
--- a/Docker/Readme.md
+++ /dev/null
@@ -1 +0,0 @@
-docker run -it   --name my-adaguc-server   --network=host   -e ADAGUC_DB="host=localhost port=5432 user=adaguc password=adaguc dbname=adaguc"   -e EXTERNALADDRESS="http://localhost:8080"   -v /data/adaguc-testsets/adaguc-data:/data/adaguc-data   -v /data/adaguc-testsets/adaguc-datasets/:/data/adaguc-datasets openearth/adaguc-server
\ No newline at end of file
diff --git a/Docker/adaguc-server-addfile.sh b/Docker/adaguc-server-addfile.sh
index da6655f83..dd51020bc 100755
--- a/Docker/adaguc-server-addfile.sh
+++ b/Docker/adaguc-server-addfile.sh
@@ -1,90 +1,3 @@
 #!/bin/bash
-
-# TODO: https://github.com/KNMI/adaguc-server/issues/71
-
-# Author: maarten.plieger@knmi.nl
-# 2020-04-17
-#
-# This script is for scanning files with adaguc-server. Adaguc can detect automatically to which dataset(s) the file belongs.
-#
-# With -f you can optionally specify the file to add
-# With -d you can optionally specify the dataset to add.
-# If only a file is specified without a dataset, adaguc-server will try to find the matching dataset that belongs to the file.
-#
-# Usage: ./adaguc-server-addfile.sh -d <datasetname (optional)> -f <file to add>
-# Usage: ./adaguc-server-addfile.sh -f <file to add> 
-# Usage: ./adaguc-server-addfile.sh -d <datasetname> 
-
-
 THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
-
-ADAGUC_DATASET=''
-
-while getopts "d:f:" o; do
-    case "${o}" in
-        d)
-            ADAGUC_DATASET=${OPTARG}
-            ;;
-        f)
-            ADAGUC_DATAFILE=${OPTARG}
-            ;;
-        h)
-            usage
-            ;;
-        *)
-            usage
-            ;;
-    esac
-done
-
-
-### Scan a file for a specified dataset ###
-
-
-if [[ -n "${ADAGUC_DATASET}" &&  -n "${ADAGUC_DATAFILE}" ]]; then
-  STATUSCODE=0
-  echo "Adding file [${ADAGUC_DATAFILE}] to dataset [${ADAGUC_DATASET}]:"
-  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${ADAGUC_DATASET} --path ${ADAGUC_DATAFILE}"
-  echo $command
-  $command
-  OUT=$?
-  if [ ${OUT} -ne 0 ]; then
-    STATUSCODE=1
-  fi
-  exit ${STATUSCODE} 
-fi
-
-
-### Scan a file ###
-if [[ -n "${ADAGUC_DATAFILE}" ]]; then
-
-  basediroffile=${ADAGUC_DATAFILE%/*}
-  basenameoffile="${ADAGUC_DATAFILE##*/}"
-  
-  STATUSCODE=0
-  echo "Adding file [${ADAGUC_DATAFILE}] to dataset [${alldatasets}]:"
-  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --autofinddataset --verboseoff --config ${ADAGUC_CONFIG} --path ${ADAGUC_DATAFILE}"
-  echo $command
-  $command
-  OUT=$?
-  if [ ${OUT} -ne 0 ]; then
-    STATUSCODE=1
-  fi
-  exit ${STATUSCODE} 
-fi
-
-# Scan a dataset
-if [[ -n "${ADAGUC_DATASET}" ]]; then
-  STATUSCODE=0
-  echo "Scanning full dataset [${ADAGUC_DATASET}]:"
-  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --verboseoff --config ${ADAGUC_CONFIG},${ADAGUC_DATASET}"
-  echo $command
-  $command
-  OUT=$?
-  if [ ${OUT} -ne 0 ]; then
-    STATUSCODE=1
-  fi
-  exit ${STATUSCODE} 
-fi
+. ${THISSCRIPTDIR}/scan.sh
diff --git a/Docker/adaguc-server-chkconfig.sh b/Docker/adaguc-server-chkconfig.sh
index 1f865e463..a961e5e68 100644
--- a/Docker/adaguc-server-chkconfig.sh
+++ b/Docker/adaguc-server-chkconfig.sh
@@ -30,8 +30,24 @@ if [ ! -f "${ADAGUC_CONFIG}" ]; then
   exit 1
 fi
 
-echo "Using adagucserver from  ${ADAGUC_PATH}"
-echo "Using config from ${ADAGUC_CONFIG}"
+### Check ADAGUC_DATA_DIR directory and set it if not set
+if [ ! -d "${ADAGUC_DATA_DIR}" ]; then
+  export ADAGUC_DATA_DIR="/data/adaguc-data"
+fi
+
+### Check ADAGUC_DATASET_DIR directory and set it if not set
+if [ ! -d "${ADAGUC_DATASET_DIR}" ]; then
+  export ADAGUC_DATASET_DIR="/data/adaguc-datasets"
+fi
+
+### Check ADAGUC_AUTOWMS_DIR directory and set it if not set
+if [ ! -d "${ADAGUC_AUTOWMS_DIR}" ]; then
+  export ADAGUC_AUTOWMS_DIR="/data/adaguc-autowms"
+fi
+
+
+# echo "Using adagucserver from  ${ADAGUC_PATH}"
+# echo "Using config from ${ADAGUC_CONFIG}"
 
 
 export ADAGUC_TMP=/tmp
diff --git a/Docker/adaguc-server-updatedatasets.sh b/Docker/adaguc-server-updatedatasets.sh
index 539bbac1a..b76d6331c 100644
--- a/Docker/adaguc-server-updatedatasets.sh
+++ b/Docker/adaguc-server-updatedatasets.sh
@@ -4,58 +4,17 @@
 # that patterns that match no files expand to zero arguments, rather than to themselves.
 shopt -s nullglob
 
-# This script requires ADAGUC_CONFIG to be set. In case of running locally with adaguc-services, this is probably
-# `export ADAGUC_CONFIG=./data/config/adaguc.vm.xml`
-
-# TODO: https://github.com/KNMI/adaguc-server/issues/71
-
 THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 . ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
 
 # Unbufferd logging for realtime output
 export ADAGUC_ENABLELOGBUFFER=FALSE
-
-
 # Do export ADAGUC_VERBOSE="--verboseon" to enable verbose logging
-
 ADAGUC_VERBOSE="${ADAGUC_VERBOSE:=--verboseoff}" 
 
 if [[ $1 ]]; then
-  # Update a specific dataset
-  for configfile in /data/adaguc-datasets/$1 ;do
-    filename=/data/adaguc-datasets/"${configfile##*/}" 
-    filebasename=${filename##*/}
-    echo ""
-    if [[ $2 ]]; then
-      echo "*** Starting update with tailpath $2 for dataset ${filename}" 
-      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} --tailpath $2
-      OUT=$?
-    else
-      echo "*** Starting update for ${filename}" 
-      ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
-      OUT=$?
-    fi
-  
-    echo ""
-  done
-
+  . ${THISSCRIPTDIR}/scan.sh -d $1
 else
- 
-  # Update all datasets
-  for configfile in /data/adaguc-datasets/*xml ;do
-    filename=/data/adaguc-datasets/"${configfile##*/}" 
-    filebasename=${filename##*/}
-    if [[ "${ADAGUC_DATASET_MASK}" && `echo ${filebasename} | grep -E ${ADAGUC_DATASET_MASK}` != ${filebasename} ]] ; then
-        if [[ "${ADAGUC_DATASET_MASK}" ]] ; then
-            echo "${filebasename} doesn't match ${ADAGUC_DATASET_MASK}"
-        fi
-        continue
-    fi
-    echo ""
-    echo "Starting update for ${filename}" 
-    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename} ${ADAGUC_VERBOSE}
-    OUT=$?
-     done
-
+. ${THISSCRIPTDIR}/scan.sh -d "*"
 fi
diff --git a/Docker/scan.sh b/Docker/scan.sh
new file mode 100755
index 000000000..75f295da6
--- /dev/null
+++ b/Docker/scan.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+#
+# This script is for scanning files with adaguc-server. Adaguc can detect automatically to which dataset(s) the file belongs.
+#
+# With -f you can optionally specify the file to add
+# With -d you can optionally specify the dataset to add.
+# If only a file is specified without a dataset, adaguc-server will try to find the matching dataset that belongs to the file.
+#
+# Usage: ./scan.sh -d <datasetname (optional)> -f <file to add>
+# Usage: ./scan.sh -f <file to add> 
+# Usage: ./scan.sh -d <datasetname> 
+# Usage: ./scan.sh -d "*" to scan all datasets
+# Usage: ./scan.sh -l to list datasets
+# Usage: ./scan.sh -e to see current environment
+
+
+THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
+
+ADAGUC_DATASET=''
+
+while getopts "d:f:le" o; do
+    case "${o}" in
+        d)
+            ADAGUC_DATASET=${OPTARG}
+            ;;
+        f)
+            ADAGUC_DATAFILE=${OPTARG}
+            ;;
+        l)
+            ls ${ADAGUC_DATASET_DIR}
+            ;;
+        e)
+            env | grep "ADAGUC"
+            ;;
+    esac
+done
+
+
+if [ $OPTIND -eq 1 ]; then ADAGUC_DATASET="*"; fi
+
+### Scan a file for a specified dataset ###
+if [[ -n "${ADAGUC_DATASET}" &&  -n "${ADAGUC_DATAFILE}" ]]; then
+  STATUSCODE=0
+  echo "Adding file [${ADAGUC_DATAFILE}] to dataset [${ADAGUC_DATASET}]:"
+  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${ADAGUC_DATASET} --path ${ADAGUC_DATAFILE}"
+  echo $command
+  $command
+  OUT=$?
+  if [ ${OUT} -ne 0 ]; then
+    STATUSCODE=1
+  fi
+  exit ${STATUSCODE} 
+fi
+
+
+### Scan a file ###
+if [[ -n "${ADAGUC_DATAFILE}" ]]; then
+
+  basediroffile=${ADAGUC_DATAFILE%/*}
+  basenameoffile="${ADAGUC_DATAFILE##*/}"
+  
+  STATUSCODE=0
+  echo "Adding file [${ADAGUC_DATAFILE}] to dataset [${alldatasets}]:"
+  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --autofinddataset --verboseoff --config ${ADAGUC_CONFIG} --path ${ADAGUC_DATAFILE}"
+  echo $command
+  $command
+  OUT=$?
+  if [ ${OUT} -ne 0 ]; then
+    STATUSCODE=1
+  fi
+  exit ${STATUSCODE} 
+fi
+
+# Scan a dataset
+if [[ -n "${ADAGUC_DATASET}" ]] && [ "${ADAGUC_DATASET}" != "*" ]; then
+  STATUSCODE=0
+  echo "Scanning full dataset [${ADAGUC_DATASET}]:"
+  command="${ADAGUC_PATH}/bin/adagucserver --updatedb --verboseoff --config ${ADAGUC_CONFIG},${ADAGUC_DATASET}"
+  echo $command
+  $command
+  OUT=$?
+  if [ ${OUT} -ne 0 ]; then
+    STATUSCODE=1
+  fi
+  exit ${STATUSCODE} 
+fi
+
+# Scan al datasets
+if [[ -n "${ADAGUC_DATASET}" ]] && [ "${ADAGUC_DATASET}" == "*" ]; then
+  echo "Scanning all datasets"
+  for configfile in ${ADAGUC_DATASET_DIR}/*xml ;do
+    STATUSCODE=0
+    echo "Scanning full dataset [${configfile}]:"
+    command="${ADAGUC_PATH}/bin/adagucserver --updatedb --verboseoff --config ${ADAGUC_CONFIG},${configfile}"
+    echo $command
+    $command
+    OUT=$?
+    if [ ${OUT} -ne 0 ]; then
+      STATUSCODE=1
+    fi
+  done
+  exit ${STATUSCODE} 
+fi
+

From 8b498ca1828bbfa4859ab7273780692d516b49a1 Mon Sep 17 00:00:00 2001
From: Maarten Plieger <maarten.plieger@knmi.nl>
Date: Wed, 23 Oct 2024 10:05:00 +0200
Subject: [PATCH 2/4] Cleanup scripts

---
 .dockerignore                                 |   3 +
 Docker/adaguc-server-addfile.sh               |   3 +-
 Docker/adaguc-server-checkfile.sh             |   9 --
 Docker/adaguc-server-cleandb.sh               | 125 ------------------
 Docker/adaguc-server-createtiles.sh           |  62 ---------
 Docker/adaguc-server-updatedatasets.sh        |  17 +--
 Docker/build.sh                               |   5 -
 Dockerfile                                    |   2 +
 python/examples/others/himawaritonetcdf.py    |   2 +-
 .../adaguc-server-chkconfig.sh                |  13 +-
 scripts/createtiles.sh                        |  26 ++++
 {Docker => scripts}/scan.sh                   |   1 -
 12 files changed, 39 insertions(+), 229 deletions(-)
 delete mode 100644 Docker/adaguc-server-checkfile.sh
 delete mode 100644 Docker/adaguc-server-cleandb.sh
 delete mode 100644 Docker/adaguc-server-createtiles.sh
 delete mode 100755 Docker/build.sh
 rename {Docker => scripts}/adaguc-server-chkconfig.sh (83%)
 mode change 100644 => 100755
 create mode 100644 scripts/createtiles.sh
 rename {Docker => scripts}/scan.sh (99%)

diff --git a/.dockerignore b/.dockerignore
index 23bfebb5b..24502d64a 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -24,3 +24,6 @@
 !tests/functional_test.py
 !tests/starttests.sh
 !tests/starttests_psql.sh
+!scripts/scan.sh
+!scripts/createtiles.sh
+!scripts/adaguc-server-chkconfig.sh
diff --git a/Docker/adaguc-server-addfile.sh b/Docker/adaguc-server-addfile.sh
index dd51020bc..c62b225f9 100755
--- a/Docker/adaguc-server-addfile.sh
+++ b/Docker/adaguc-server-addfile.sh
@@ -1,3 +1,2 @@
 #!/bin/bash
-THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-. ${THISSCRIPTDIR}/scan.sh
+. ${ADAGUC_PATH}/scripts/scan.sh
diff --git a/Docker/adaguc-server-checkfile.sh b/Docker/adaguc-server-checkfile.sh
deleted file mode 100644
index 4342752ae..000000000
--- a/Docker/adaguc-server-checkfile.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-export ADAGUC_PATH=/adaguc/adaguc-server-master/
-export ADAGUC_TMP=/tmp
-export ADAGUC_ONLINERESOURCE=""
-export CHECKERFILE="/tmp/"`uuidgen`.report
-export ADAGUC_CONFIG=/adaguc/adaguc-server-config.xml
-export QUERY_STRING="source=$1&&service=WMS&request=getcapabilities"
-/adaguc/adaguc-server-master/bin/adagucserver --report=$CHECKERFILE > /dev/null
-cat $CHECKERFILE
\ No newline at end of file
diff --git a/Docker/adaguc-server-cleandb.sh b/Docker/adaguc-server-cleandb.sh
deleted file mode 100644
index 0c75acc66..000000000
--- a/Docker/adaguc-server-cleandb.sh
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/bin/bash
-
-# Author: maarten.plieger@knmi.nl
-# 2020-04-23
-# This script cleans the adaguc database for files which are older than specified nr of days
-
-#Usage: bash adaguc-server-cleandb.sh -p <directory> -f <filter> -d <number of days> -q filetimedate -t <delete_db or delete_db_and_fs>
-#Example: bash adaguc-server-cleandb.sh  -p /data/adaguc-autowms/EGOWS_radar -f ".*\.nc$" -d 7 -q filetimedate -t delete_db
-
-# The script should only print the deleted files.
-
-usage () {
-    echo "This script cleans the database for files which are older than specified nr of days"
-    echo "-p: is for the filepath as configured in your Layers FilePath value"
-    echo "-f: is for the filefilter as configured in your Layers FilePath filter attribute. Can be set to * to ignore or left out"
-    echo "-d: Specify how many days old the files need to be for removal"
-    echo "-q: Querytype, currently filetimedate, this is the date inside the time variable of the NetCDF file"
-    echo "-t: Deletetype, delete_db means it will be removed from the db only, delete_db_and_fs will also delete the files from disk"
-    exit
-}
-
-
-while getopts "p:f:d:q:t:" o; do
-    case "${o}" in
-        p)
-            FILEPATH=${OPTARG}
-            ;;
-        f)
-            FILEFILTER=${OPTARG}
-            ;;
-        d)
-            FILEDAYS=${OPTARG}
-            ;;
-        q)
-            QUERYTYPE=${OPTARG}
-            ;;
-        t)
-            DELETEMODE=${OPTARG}
-            ;;
-        h)
-            usage
-            ;;
-        *)
-            usage
-            ;;
-    esac
-done
-
-if [ ! -n "${FILEPATH}" ]; then
-  echo "You did not supply a filepath with -p <filepath>"
-  exit 1
-fi
-
-if [ ! -n "${FILEDAYS}" ]; then
-  echo "You did not supply the days argument with -d <nr of days>"
-  exit 1
-fi
-
-if [ ! -n "${QUERYTYPE}" ]; then
-  echo "You did not supply the QUERYTYPE argument with -q filetimedate"
-  exit 1
-fi
-
-if [ ! -n "${DELETEMODE}" ]; then
-  echo "You did not supply the delete mode argument with -t <delete_db or delete_db_and_fs>"
-  exit 1
-fi
-
-    
-THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
-
-
-datadir=${FILEPATH} #, as configured in your dataset
-datafilter=${FILEFILTER:-""}
-limitdaysold=${FILEDAYS}
-
-# Cut trailing slash
-datadir=${datadir%/} 
-
-pathfiltertablelookuptable="pathfiltertablelookup_v2_0_23"
-
-defaultpostgrescredentials="host=localhost port=5432 user=adaguc password=adaguc dbname=adaguc" # From adaguc-server-config.xml
-
-postgrescredentials=${ADAGUC_DB:-${defaultpostgrescredentials}}
-
-
-encodeddatafilter="and filter = E'F_${datafilter}'"
-
-if [[ "${datafilter}" == "*" ]];then
-encodeddatafilter=""
-fi
-
-if [[ "${datafilter}" == "" ]];then
-encodeddatafilter=""
-fi
-#echo "select tablename from ${pathfiltertablelookuptable} where path = E'P_${datadir}' ${encodeddatafilter} and dimension!='time';"
-# Get the right tablenames from the database, based on the directory.
-tablenames=$(psql -t "${postgrescredentials}" -c "select tablename from ${pathfiltertablelookuptable} where path = E'P_${datadir}' ${encodeddatafilter} and dimension!='time';")
-timetablename=$(psql -t "${postgrescredentials}" -c "select tablename from ${pathfiltertablelookuptable} where path = E'P_${datadir}' ${encodeddatafilter} and dimension='time';")
-
-# Get the list of files with more than $limitdaysold days old.
-timeolder=`date --date="${limitdaysold} days ago" +%Y-%m-%d`T00:00:00Z
-filelist=$(psql -t "${postgrescredentials}" -c "select path from ${timetablename} where time < '${timeolder}' order by time asc;")
-
-
-# Delete the files from the FS
-if [ ${DELETEMODE} == "delete_db_and_fs" ]; then
-  for f in $filelist; do
-    rm -f $f
-  done
-fi
-
-function join_by { local d=$1; shift; echo -n "$1"; shift; printf "%s" "${@/#/$d}"; }
-
-echo {\"filelist\":[\"`join_by "\",\"" ${filelist}`\"]}
-
-# First delete from all other dims
-for tablename in $tablenames;do
-  psql -t "${postgrescredentials}" -c "delete from ${tablename} where path in (select path from ${timetablename} where time < '${timeolder}' order by time asc);" >/dev/null
-done
-
-# Finally delete also from time dim
-psql -t "${postgrescredentials}" -c "delete from ${timetablename} where path in (select path from ${timetablename} where time < '${timeolder}' order by time asc);" >/dev/null
diff --git a/Docker/adaguc-server-createtiles.sh b/Docker/adaguc-server-createtiles.sh
deleted file mode 100644
index da463c8fa..000000000
--- a/Docker/adaguc-server-createtiles.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/bin/bash
-#!/bin/bash
-
-# TODO: https://github.com/KNMI/adaguc-server/issues/71
-
-DOCKER_ADAGUC_PATH=/adaguc/adaguc-server-master
-DOCKER_ADAGUC_CONFIG=/adaguc/adaguc-server-config.xml
-
-THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-### Check if ADAGUC_PATH is set externally, if not set it to default ###
-if [ ! -f "${ADAGUC_PATH}/bin/adagucserver" ]; then
-  export ADAGUC_PATH=${DOCKER_ADAGUC_PATH}
-fi
-
-### Check if we can find adaguc executable at default location, otherwise try it from this location ###
-if [ ! -f "${ADAGUC_PATH}/bin/adagucserver" ]; then
-  export ADAGUC_PATH=${THISSCRIPTDIR}/../
-fi
-
-### Check if we could find adaguc executable ###
-if [ ! -f "${ADAGUC_PATH}/bin/adagucserver" ]; then
-  >&2 echo "No adagucserver executable found in path ADAGUC_PATH/bin/adagucserver [${ADAGUC_PATH}/bin/adagucserver] "
-  exit 1
-fi
-
-### Check configuratiion file location ###
-if [ ! -f "${ADAGUC_CONFIG}" ]; then
-  export ADAGUC_CONFIG=${DOCKER_ADAGUC_CONFIG}
-fi
-
-### Checks if configuration file exists
-if [ ! -f "${ADAGUC_CONFIG}" ]; then
-  >&2 echo "No configuration file found ADAGUC_CONFIG variable [${ADAGUC_CONFIG}] "
-  exit 1
-fi
-
-echo "Using adagucserver from  ${ADAGUC_PATH}"
-echo "Using config from ${ADAGUC_CONFIG}"
-
-
-export ADAGUC_TMP=/tmp
-export ADAGUC_ONLINERESOURCE=""
-
-if [[ $1 ]]; then
-  # Update a specific dataset
-  for configfile in /data/adaguc-datasets/$1 ;do
-    filename=/data/adaguc-datasets/"${configfile##*/}" 
-    filebasename=${filename##*/}
-    echo ""
-    echo "*** Starting updatedb for ${filename}" 
-    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename}
-    echo "*** Starting create tiles for ${filename}" 
-    ${ADAGUC_PATH}/bin/adagucserver --createtiles --config ${ADAGUC_CONFIG},${filename}
-    OUT=$?
-  done
-
-else
-  echo "Please specify a dataset"
-fi
-
-
diff --git a/Docker/adaguc-server-updatedatasets.sh b/Docker/adaguc-server-updatedatasets.sh
index b76d6331c..58434e17f 100644
--- a/Docker/adaguc-server-updatedatasets.sh
+++ b/Docker/adaguc-server-updatedatasets.sh
@@ -1,20 +1,7 @@
 #!/bin/bash
-
-# nullglob is a Bash shell option which modifies [[glob]] expansion such
-# that patterns that match no files expand to zero arguments, rather than to themselves.
 shopt -s nullglob
-
-THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
-
-# Unbufferd logging for realtime output
-export ADAGUC_ENABLELOGBUFFER=FALSE
-# Do export ADAGUC_VERBOSE="--verboseon" to enable verbose logging
-ADAGUC_VERBOSE="${ADAGUC_VERBOSE:=--verboseoff}" 
-
 if [[ $1 ]]; then
-  . ${THISSCRIPTDIR}/scan.sh -d $1
+  . ${ADAGUC_PATH}/scripts/scan.sh -d $1
 else
-. ${THISSCRIPTDIR}/scan.sh -d "*"
+. ${ADAGUC_PATH}/scripts/scan.sh -d "*"
 fi
diff --git a/Docker/build.sh b/Docker/build.sh
deleted file mode 100755
index 9447f7a16..000000000
--- a/Docker/build.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-echo "Building build adaguc-server"
-docker build -t adaguc-server ..
-#echo "Exporting image"
-#docker save -o adaguc-server.dockerimage adaguc-server
diff --git a/Dockerfile b/Dockerfile
index d10e0111a..8ecaefe42 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -123,6 +123,7 @@ COPY ./Docker/adaguc-server-config-python-postgres.xml /adaguc/adaguc-server-con
 COPY ./Docker/start.sh /adaguc/
 COPY ./Docker/adaguc-server-*.sh /adaguc/
 COPY ./Docker/baselayers.xml /adaguc/adaguc-datasets-internal/baselayers.xml
+COPY ./scripts/*.sh /adaguc/adaguc-server-master/scripts/
 # Copy pgbouncer and supervisord config files
 COPY ./Docker/pgbouncer/ /adaguc/pgbouncer/
 COPY ./Docker/supervisord/ /etc/supervisor/conf.d/
@@ -130,6 +131,7 @@ COPY ./Docker/run_supervisord.sh /adaguc/run_supervisord.sh
 # Set permissions
 RUN  chmod +x /adaguc/adaguc-server-*.sh && \
     chmod +x /adaguc/start.sh && \
+    chmod +x /adaguc/adaguc-server-master/scripts/*.sh && \
     chown -R adaguc:adaguc /data/adaguc* /adaguc /adaguc/*
 
 ENV ADAGUC_PATH=/adaguc/adaguc-server-master
diff --git a/python/examples/others/himawaritonetcdf.py b/python/examples/others/himawaritonetcdf.py
index f09015917..3ee5b6a78 100644
--- a/python/examples/others/himawaritonetcdf.py
+++ b/python/examples/others/himawaritonetcdf.py
@@ -169,7 +169,7 @@ def convert(self, mfile):
         if os.path.isfile(outputname) == False:
             try:
                 self.convertonc(inputname,mfile, self.imgextent,self.imgproj,self.product,outputname)  
-                subprocess.call(("docker exec -i -t my-adaguc-server bash /adaguc/adaguc-server-createtiles.sh " + self.adagucdatasetname).split())
+                subprocess.call(("docker exec -i -t my-adaguc-server bash /adaguc/adaguc-server-master/scripts/createtiles.sh " + self.adagucdatasetname).split())
             except:
                 print "Error converting image %s" % inputname
                 os.remove(inputname)
diff --git a/Docker/adaguc-server-chkconfig.sh b/scripts/adaguc-server-chkconfig.sh
old mode 100644
new mode 100755
similarity index 83%
rename from Docker/adaguc-server-chkconfig.sh
rename to scripts/adaguc-server-chkconfig.sh
index a961e5e68..f5b020549
--- a/Docker/adaguc-server-chkconfig.sh
+++ b/scripts/adaguc-server-chkconfig.sh
@@ -1,11 +1,11 @@
-DOCKER_ADAGUC_PATH=/adaguc/adaguc-server-master
-DOCKER_ADAGUC_CONFIG=/adaguc/adaguc-server-config.xml
+DEFAULT_ADAGUC_PATH=/adaguc/adaguc-server-master
+DEFAULT_ADAGUC_CONFIG=/adaguc/adaguc-server-config.xml
 
 THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 ### Check if ADAGUC_PATH is set externally, if not set it to default ###
 if [ ! -f "${ADAGUC_PATH}/bin/adagucserver" ]; then
-  export ADAGUC_PATH=${DOCKER_ADAGUC_PATH}
+  export ADAGUC_PATH=${DEFAULT_ADAGUC_PATH}
 fi
 
 ### Check if we can find adaguc executable at default location, otherwise try it from this location ###
@@ -21,7 +21,7 @@ fi
 
 ### Check configuratiion file location ###
 if [ ! -f "${ADAGUC_CONFIG}" ]; then
-  export ADAGUC_CONFIG=${DOCKER_ADAGUC_CONFIG}
+  export ADAGUC_CONFIG=${DEFAULT_ADAGUC_CONFIG}
 fi
 
 ### Checks if configuration file exists
@@ -45,10 +45,5 @@ if [ ! -d "${ADAGUC_AUTOWMS_DIR}" ]; then
   export ADAGUC_AUTOWMS_DIR="/data/adaguc-autowms"
 fi
 
-
-# echo "Using adagucserver from  ${ADAGUC_PATH}"
-# echo "Using config from ${ADAGUC_CONFIG}"
-
-
 export ADAGUC_TMP=/tmp
 export ADAGUC_ONLINERESOURCE=""
diff --git a/scripts/createtiles.sh b/scripts/createtiles.sh
new file mode 100644
index 000000000..5d97b4545
--- /dev/null
+++ b/scripts/createtiles.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
+
+echo "Using adagucserver from  ${ADAGUC_PATH}"
+echo "Using config from ${ADAGUC_CONFIG}"
+
+if [[ $1 ]]; then
+  # Update a specific dataset
+  for configfile in /data/adaguc-datasets/$1 ;do
+    filename=/data/adaguc-datasets/"${configfile##*/}" 
+    filebasename=${filename##*/}
+    echo ""
+    echo "*** Starting updatedb for ${filename}" 
+    ${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},${filename}
+    echo "*** Starting create tiles for ${filename}" 
+    ${ADAGUC_PATH}/bin/adagucserver --createtiles --config ${ADAGUC_CONFIG},${filename}
+    OUT=$?
+  done
+
+else
+  echo "Please specify a dataset"
+fi
+
+
diff --git a/Docker/scan.sh b/scripts/scan.sh
similarity index 99%
rename from Docker/scan.sh
rename to scripts/scan.sh
index 75f295da6..2f0e132c6 100755
--- a/Docker/scan.sh
+++ b/scripts/scan.sh
@@ -15,7 +15,6 @@
 
 
 THISSCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
 . ${THISSCRIPTDIR}/adaguc-server-chkconfig.sh
 
 ADAGUC_DATASET=''

From 46d6fa6cbca0b70722deb2c355db50f9c22f218b Mon Sep 17 00:00:00 2001
From: Maarten Plieger <maarten.plieger@knmi.nl>
Date: Wed, 23 Oct 2024 10:30:57 +0200
Subject: [PATCH 3/4] Cleanup

---
 .dockerignore             |  4 +---
 Dockerfile                |  6 +++---
 doc/Developing.md         |  4 +++-
 doc/overview/QuickTips.md | 23 +++++++++++++++++++----
 4 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 24502d64a..dd0acb354 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -17,6 +17,7 @@
 !cmake/
 !data/
 !hclasses/
+!scripts/
 !python/
 !tests/AdagucTests/
 !tests/expectedoutputs/
@@ -24,6 +25,3 @@
 !tests/functional_test.py
 !tests/starttests.sh
 !tests/starttests_psql.sh
-!scripts/scan.sh
-!scripts/createtiles.sh
-!scripts/adaguc-server-chkconfig.sh
diff --git a/Dockerfile b/Dockerfile
index 8ecaefe42..5f4972b1b 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -123,14 +123,14 @@ COPY ./Docker/adaguc-server-config-python-postgres.xml /adaguc/adaguc-server-con
 COPY ./Docker/start.sh /adaguc/
 COPY ./Docker/adaguc-server-*.sh /adaguc/
 COPY ./Docker/baselayers.xml /adaguc/adaguc-datasets-internal/baselayers.xml
-COPY ./scripts/*.sh /adaguc/adaguc-server-master/scripts/
+COPY scripts /adaguc/adaguc-server-master/scripts
+COPY ./scripts/*.sh /adaguc/
 # Copy pgbouncer and supervisord config files
 COPY ./Docker/pgbouncer/ /adaguc/pgbouncer/
 COPY ./Docker/supervisord/ /etc/supervisor/conf.d/
 COPY ./Docker/run_supervisord.sh /adaguc/run_supervisord.sh
 # Set permissions
-RUN  chmod +x /adaguc/adaguc-server-*.sh && \
-    chmod +x /adaguc/start.sh && \
+RUN  chmod +x /adaguc/*.sh && \
     chmod +x /adaguc/adaguc-server-master/scripts/*.sh && \
     chown -R adaguc:adaguc /data/adaguc* /adaguc /adaguc/*
 
diff --git a/doc/Developing.md b/doc/Developing.md
index bad0ba79c..a8801f3bb 100644
--- a/doc/Developing.md
+++ b/doc/Developing.md
@@ -146,4 +146,6 @@ bash runtests.sh
 
 ## To scan datasets
 
-`bash ./Docker/adaguc-server-updatedatasets.sh <dataset name>`
\ No newline at end of file
+
+- list datasets: `bash ./scripts/scan.sh -l`
+- scan a dataset: `bash ./scripts/scan.sh -d <dataset name>`
\ No newline at end of file
diff --git a/doc/overview/QuickTips.md b/doc/overview/QuickTips.md
index 9d7496ad1..4bc7b7e31 100644
--- a/doc/overview/QuickTips.md
+++ b/doc/overview/QuickTips.md
@@ -4,9 +4,24 @@
 
 ## Scanning datasets
 
-Scan with the adaguc-server container:
+List datasets: 
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh <optional datasetname>
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -l
+```
+
+Scan a dataset:
+```
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d <datasetname>
+```
+
+Scan a file:
+```
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -f <filename to scan>
+```
+
+Scan all datasets: 
+```
+docker exec -i -t my-adaguc-server /adaguc/scan.sh
 ```
 
 Similar command, but in this case we go inside the container and call the script directly:
@@ -14,8 +29,8 @@ Similar command, but in this case we go inside the container and call the script
 ```
 docker exec -it my-adaguc-server bash
 cd /adaguc
-. ./adaguc-server-chkconfig.sh
-${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},<optional datasetname>
+. adaguc-server-chkconfig.sh
+${ADAGUC_PATH}/bin/adagucserver --updatedb --config ${ADAGUC_CONFIG},<datasetname>
 ```
 
 

From ac1e7146ce4bf21f0aef6191e33b59dd0a64b0c2 Mon Sep 17 00:00:00 2001
From: Maarten Plieger <maarten.plieger@knmi.nl>
Date: Wed, 23 Oct 2024 10:55:10 +0200
Subject: [PATCH 4/4] Updated documentation

---
 doc/Running.md                                         |  3 ++-
 doc/info/Docker.md                                     |  6 ++----
 doc/tutorials/Configure_EDR_service.md                 |  2 +-
 .../Create_WMS_on_series_KNMI_HDF5_radar_files.md      |  2 +-
 ...rvice_on_a_series_of_files_with_a_time_dimension.md | 10 +++++-----
 5 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/doc/Running.md b/doc/Running.md
index de612e31f..2c80b5322 100644
--- a/doc/Running.md
+++ b/doc/Running.md
@@ -65,7 +65,8 @@ If you now see the viewer, you have succesfully started the adaguc-server.
 
 Scan with the adaguc-server container:
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh <optional datasetname>
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -l
+docker exec -i -t my-adaguc-server /adaguc/scan.sh <optional -d datasetname>
 ```
 
 ## Visit the webservice:
diff --git a/doc/info/Docker.md b/doc/info/Docker.md
index 8df0df749..7fb212fe0 100644
--- a/doc/info/Docker.md
+++ b/doc/info/Docker.md
@@ -28,8 +28,7 @@ docker-compose build
 docker-compose up -d && sleep 10
 
 1.  Scan datasets:
-    docker exec -i -t my-adaguc-server
-    /adaguc/adaguc-server-updatedatasets.sh
+    docker exec -i -t my-adaguc-server /adaguc/scan.sh
 
 <!-- -->
 
@@ -104,8 +103,7 @@ mkdir \$HOME/adaguc-datasets
 <!-- -->
 
 1.  Scan datasets:
-    docker exec -i -t my-adaguc-server
-    /adaguc/adaguc-server-updatedatasets.sh
+    docker exec -i -t my-adaguc-server /adaguc/scan.sh
 
 <!-- -->
 
diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md
index 08276760c..6967681c1 100644
--- a/doc/tutorials/Configure_EDR_service.md
+++ b/doc/tutorials/Configure_EDR_service.md
@@ -108,7 +108,7 @@ parameter_names": {
 ## Step 3: Scan the new data
 
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh edr
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d edr
 ```
 
 ## Step 4: Check if the EDR endpoint works
diff --git a/doc/tutorials/Create_WMS_on_series_KNMI_HDF5_radar_files.md b/doc/tutorials/Create_WMS_on_series_KNMI_HDF5_radar_files.md
index f28126221..1fe22ba2c 100644
--- a/doc/tutorials/Create_WMS_on_series_KNMI_HDF5_radar_files.md
+++ b/doc/tutorials/Create_WMS_on_series_KNMI_HDF5_radar_files.md
@@ -130,7 +130,7 @@ This file is called a dataset configuration and is normally stored in the adaguc
 # Step 4: Scan the files
 
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh RAD_NL25_PCP_CM
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d RAD_NL25_PCP_CM
 ```
 
 ## Output:
diff --git a/doc/tutorials/Create_a_WMS_service_on_a_series_of_files_with_a_time_dimension.md b/doc/tutorials/Create_a_WMS_service_on_a_series_of_files_with_a_time_dimension.md
index 5ed4fc8d9..7efc70c99 100644
--- a/doc/tutorials/Create_a_WMS_service_on_a_series_of_files_with_a_time_dimension.md
+++ b/doc/tutorials/Create_a_WMS_service_on_a_series_of_files_with_a_time_dimension.md
@@ -31,13 +31,13 @@ You could copy these files from [here - python/examples/rundataset/data/sequence
 When everything is configured, you are ready to scan (or ingest) the files, this can be done with the following command:
 
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh mymonthlyconfig
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d mymonthlyconfig
 ```
 
 It should print something like:
 
 ```
-$ docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh mymonthlyconfig
+$ docker exec -i -t my-adaguc-server /adaguc/scan.sh -d mymonthlyconfig
 Using adagucserver from  /adaguc/adaguc-server-master
 Using config from /adaguc/adaguc-server-config.xml
 
@@ -76,13 +76,13 @@ If you would like to add a single file in the configured data directory,
 you can do:
 
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh mymonthlyconfig apr.nc
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d mymonthlyconfig apr.nc
 ```
 
 If you would like to scan specific files in a certain subdirectory of
 the configured data directory, you can do:
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh mymonthlyconfig 2013
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d mymonthlyconfig 2013
 ```
 
 This adds the files which are located in the given subdirectory. If
@@ -95,7 +95,7 @@ If you would like to remove files from the service, because they do not
 exist anymore on the filesystem or for any other reason, you can call
 updatedb again with only the dataset:
 ```
-docker exec -i -t my-adaguc-server /adaguc/adaguc-server-updatedatasets.sh mymonthlyconfig
+docker exec -i -t my-adaguc-server /adaguc/scan.sh -d mymonthlyconfig
 ```
 
 This checks if files are present in adaguc-server database but not on the filesystem. It will also check if files have been updated.