Merge branch 'main' into feature/datashield

bbmri/docker-compose.yml

@@ -8,8 +8,9 @@ services:
     container_name: bridgehead-bbmri-blaze
     environment:
       BASE_URL: "http://bridgehead-bbmri-blaze:8080"
-      JAVA_TOOL_OPTIONS: "-Xmx4g"
-      LOG_LEVEL: "debug"
+      JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
+      DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
+      DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
       - "blaze-data:/app/data"

bridgehead

@@ -51,6 +51,7 @@ loadVars() {
 	fi
 	fetchVarsFromVaultByFile /etc/bridgehead/$PROJECT.conf || fail_and_report 1 "Unable to fetchVarsFromVaultByFile"
 	setHostname
+	optimizeBlazeMemoryUsage
 	[ -e ./$PROJECT/vars ] && source ./$PROJECT/vars
 	set +a
 

ccp/docker-compose.yml

@@ -6,7 +6,9 @@ services:
     container_name: bridgehead-ccp-blaze
     environment:
       BASE_URL: "http://bridgehead-ccp-blaze:8080"
-      JAVA_TOOL_OPTIONS: "-Xmx4g"
+      JAVA_TOOL_OPTIONS: "-Xmx${BLAZE_MEMORY_CAP:-4096}m"
+      DB_RESOURCE_CACHE_SIZE: ${BLAZE_RESOURCE_CACHE_CAP:-2500000}
+      DB_BLOCK_CACHE_SIZE: $BLAZE_MEMORY_CAP
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
       - "blaze-data:/app/data"
@@ -19,7 +21,7 @@ services:
       - "traefik.http.routers.blaze_ccp.tls=true"
 
   focus:
-    image: docker.verbis.dkfz.de/cache/samply/focus:0.4.0
+    image: docker.verbis.dkfz.de/cache/samply/focus:0.4.1
     container_name: bridgehead-focus
     environment:
       API_KEY: ${FOCUS_BEAM_SECRET_SHORT}

ccp/modules/dnpm-node-compose.yml

@@ -6,6 +6,7 @@ services:
     container_name: bridgehead-dnpm-backend
     environment:
       - ZPM_SITE=${ZPM_SITE}
+      - N_RANDOM_FILES=${DNPM_SYNTH_NUM}
     volumes:
       - /etc/bridgehead/dnpm:/bwhc_config:ro
       - ${DNPM_DATA_DIR}:/bwhc_data

ccp/modules/dnpm-node-setup.sh

@@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then
 		log ERROR "Mandatory variable DNPM_DATA_DIR not defined!"
 		exit 1
 	fi
-			if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then
-				echo "Override of landing page url already in place"
-			else
-				echo "Adding override of landing page url"
-				if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then
-					echo -e '  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
-				else
-					echo -e 'version: "3.7"\nservices:\n  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
-				fi
-			fi
+	DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0}
+	if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then
+		echo "Override of landing page url already in place"
+	else
+		echo "Adding override of landing page url"
+		if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then
+			echo -e '  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
+		else
+			echo -e 'version: "3.7"\nservices:\n  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
+		fi
+	fi
 fi

lib/functions.sh

@@ -155,6 +155,28 @@ setHostname() {
 	fi
 }
 
+# This function optimizes the usage of memory through blaze, according to the official performance tuning guide:
+#   https://github.com/samply/blaze/blob/master/docs/tuning-guide.md
+# Short summary of the adjustments made:
+# - set blaze memory cap to a quarter of the system memory
+# - set db block cache size to a quarter of the system memory
+# - limit resource count allowed in blaze to 1,25M per 4GB available system memory
+optimizeBlazeMemoryUsage() {
+	if [ -z "$BLAZE_MEMORY_CAP" ]; then
+	   system_memory_in_mb=$(LC_ALL=C free -m | grep 'Mem:' | awk '{print $2}');
+	   export BLAZE_MEMORY_CAP=$(("$system_memory_in_mb"/4));
+	fi
+	if [ -z "$BLAZE_RESOURCE_CACHE_CAP" ]; then
+		available_system_memory_chuncks=$((BLAZE_MEMORY_CAP / 1000))
+		if [ $available_system_memory_chuncks -eq 0 ]; then
+			log WARN "Only ${BLAZE_MEMORY_CAP} system memory available for Blaze. If your Blaze stores more than 128000 fhir ressources it will run significally slower."
+			export BLAZE_RESOURCE_CACHE_CAP=128000;
+		else
+			export BLAZE_RESOURCE_CACHE_CAP=$((available_system_memory_chuncks * 312500))
+		fi
+	fi
+}
+
 # Takes 1) The Backup Directory Path 2) The name of the Service to be backuped
 # Creates 3 Backups: 1) For the past seven days 2) For the current month and 3) for each calendar week
 createEncryptedPostgresBackup(){

lib/prerequisites.sh

@@ -67,29 +67,30 @@ log INFO "Checking network access ($BROKER_URL_FOR_PREREQ) ..."
 source /etc/bridgehead/${PROJECT}.conf
 source ${PROJECT}/vars
 
-set +e
-SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
-RET=$?
-set -e
-if [ $RET -ne 0 ]; then
-	log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication."
-	log WARN "Unable to check clock skew due to previous error."
-else
-	log INFO "Checking clock skew ..."
-
-	SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s)
-	MYTIME=$(date +%s)
-	SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME))
-	SKEW=$(echo $SKEW | awk -F- '{print $NF}')
-	SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd."
-	if [ $SKEW -ge 300 ]; then
-		report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT"
-		exit 1
-	elif [ $SKEW -ge 60 ]; then
-		log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT"
-	fi
+if [ "${PROJECT}" != "minimal" ]; then
+  set +e
+  SERVERTIME="$(https_proxy=$HTTPS_PROXY_FULL_URL curl -m 5 -s -I $BROKER_URL_FOR_PREREQ 2>&1 | grep -i -e '^Date: ' | sed -e 's/^Date: //i')"
+  RET=$?
+  set -e
+  if [ $RET -ne 0 ]; then
+	  log WARN "Unable to connect to Samply.Beam broker at $BROKER_URL_FOR_PREREQ. Please check your proxy settings.\nThe currently configured proxy was \"$HTTPS_PROXY_URL\". This error is normal when using proxy authentication."
+  	log WARN "Unable to check clock skew due to previous error."
+  else
+	  log INFO "Checking clock skew ..."
+
+    SERVERTIME_AS_TIMESTAMP=$(date --date="$SERVERTIME" +%s)
+	  MYTIME=$(date +%s)
+	  SKEW=$(($SERVERTIME_AS_TIMESTAMP - $MYTIME))
+	  SKEW=$(echo $SKEW | awk -F- '{print $NF}')
+	  SYNCTEXT="For example, consider entering a correct NTP server (e.g. your institution's Active Directory Domain Controller in /etc/systemd/timesyncd.conf (option NTP=) and restart systemd-timesyncd."
+	  if [ $SKEW -ge 300 ]; then
+		  report_error 5 "Your clock is not synchronized (${SKEW}s off). This will cause Samply.Beam's certificate will fail. Please setup time synchronization. $SYNCTEXT"
+		  exit 1
+	  elif [ $SKEW -ge 60 ]; then
+		  log WARN "Your clock is more than a minute off (${SKEW}s). Consider syncing to a time server. $SYNCTEXT"
+	  fi
+  fi
 fi
-
 checkPrivKey() {
   if [ -e /etc/bridgehead/pki/${SITE_ID}.priv.pem ]; then
     log INFO "Success - private key found."
@@ -100,7 +101,7 @@ checkPrivKey() {
   return 0
 }
 
-if [[ "$@" =~ "noprivkey" ]]; then
+if [[ "$@" =~ "noprivkey" ||  "${PROJECT}" != "minimal" ]]; then
   log INFO "Skipping check for private key for now."
 else
   checkPrivKey || exit 1

minimal/modules/dnpm-node-compose.yml

@@ -6,6 +6,7 @@ services:
     container_name: bridgehead-dnpm-backend
     environment:
       - ZPM_SITE=${ZPM_SITE}
+      - N_RANDOM_FILES=${DNPM_SYNTH_NUM}
     volumes:
       - /etc/bridgehead/dnpm:/bwhc_config:ro
       - ${DNPM_DATA_DIR}:/bwhc_data

minimal/modules/dnpm-node-setup.sh

@@ -14,14 +14,15 @@ if [ -n "${ENABLE_DNPM_NODE}" ]; then
 		log ERROR "Mandatory variable DNPM_DATA_DIR not defined!"
 		exit 1
 	fi
-			if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then
-				echo "Override of landing page url already in place"
-			else
-				echo "Adding override of landing page url"
-				if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then
-					echo -e '  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
-				else
-					echo -e 'version: "3.7"\nservices:\n  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
-				fi
-			fi
+	DNPM_SYNTH_NUM=${DNPM_SYNTH_NUM:-0}
+	if grep -q 'traefik.http.routers.landing.rule=PathPrefix(`/landing`)' /srv/docker/bridgehead/minimal/docker-compose.override.yml 2>/dev/null; then
+		echo "Override of landing page url already in place"
+	else
+		echo "Adding override of landing page url"
+		if [ -f /srv/docker/bridgehead/minimal/docker-compose.override.yml ]; then
+			echo -e '  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
+		else
+			echo -e 'version: "3.7"\nservices:\n  landing:\n    labels:\n      - "traefik.http.routers.landing.rule=PathPrefix(`/landing`)"' >> /srv/docker/bridgehead/minimal/docker-compose.override.yml
+		fi
+	fi
 fi