Merge pull request #5 from ITISFoundation/batch_mode

Batch mode

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.0.12
+current_version = 0.0.14
 commit = False
 message = service version: {current_version} → {new_version}
 tag = False

.osparc/osparc-meta-parallelrunner/metadata.yml

@@ -1,7 +1,7 @@
 name: Parallel Runner
 description: "ParallelRunnerService"
 key: simcore/services/dynamic/osparc-meta-parallelrunner
-version: 0.0.12
+version: 0.0.14
 integration-version: 2.0.0
 type: dynamic
 authors:
@@ -28,6 +28,12 @@ inputs:
     description:
       File with the parameter sets to evaluate
     type: data:*/*
+  input_3:
+    displayOrder: 2.0
+    label: Settings
+    description:
+      JSON file with settings for the parallel runner
+    type: data:*/*
 outputs:
   output_1:
     displayOrder: 1.0

Dockerfile

@@ -15,8 +15,7 @@ RUN apt-get install --yes --no-install-recommends python3 python-is-python3 pyth
 # Copying boot scripts                                                                                                                                                                                                                                                                                                   
 COPY docker_scripts /docker
 
-RUN pip3 install pathos --upgrade
-RUN pip3 install /docker/*.whl
+RUN pip3 install pathos osparc pydantic-settings osparc-filecomms --upgrade
 
 USER osparcuser
 

Makefile

@@ -4,8 +4,8 @@ SHELL = /bin/sh
 .DEFAULT_GOAL := help
 MAKEFLAGS += -j2
 
-export DOCKER_IMAGE_NAME ?= osparc-map
-export DOCKER_IMAGE_TAG ?= 0.0.12
+export DOCKER_IMAGE_NAME ?= osparc-meta-parallelrunner
+export DOCKER_IMAGE_TAG ?= 0.0.14
 
 export MASTER_AWS_REGISTRY ?= registry.osparc-master-zmt.click
 export MASTER_REGISTRY ?= registry.osparc-master.speag.com

docker-compose-local.yml

@@ -1,7 +1,7 @@
 version: '3.7'
 services:
   osparc-meta-parallelrunner:
-    image: simcore/services/dynamic/osparc-meta-parallelrunner:0.0.12
+    image: simcore/services/dynamic/osparc-meta-parallelrunner:0.0.14
     ports:
       - "8888:8888"
     environment:

docker_scripts/entrypoint.bash

@@ -4,40 +4,37 @@ set -euo pipefail
 IFS=$'\n\t'
 INFO="INFO: [$(basename "$0")] "
 
-echo "$INFO" "Starting container for map ..."
+echo "$INFO" "Starting container for parallelrunner ..."
 
 HOST_USERID=$(stat -c %u "${DY_SIDECAR_PATH_INPUTS}")
 HOST_GROUPID=$(stat -c %g "${DY_SIDECAR_PATH_INPUTS}")
 CONTAINER_GROUPNAME=$(getent group | grep "${HOST_GROUPID}" | cut --delimiter=: --fields=1 || echo "")
 
-
 OSPARC_USER='osparcuser'
 
-if [ "$HOST_USERID" -eq 0 ]                                                                                                                                                                                                        
-then                                                                                                                                                                                                                               
-    echo "Warning: Folder mounted owned by root user... adding $OSPARC_USER to root..."                                                                                                                                                
-    addgroup "$OSPARC_USER" root                                                                                                                                                                                                       
-else                                                                                                                                                                                                                               
-    echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONTAINER_GROUPNAME'..."                                                                                                                                       
-    # take host's credentials in $OSPARC_USER                                                                                                                                                                                          
-    if [ -z "$CONTAINER_GROUPNAME" ]                                                                                                                                                                                               
-    then                                                                                                                                                                                                                           
-        echo "Creating new group my$OSPARC_USER"                                                                                                                                                                                       
-        CONTAINER_GROUPNAME=my$OSPARC_USER                                                                                                                                                                                             
-        addgroup --gid "$HOST_GROUPID" "$CONTAINER_GROUPNAME"                                                                                                                                                                      
-    else                                                                                                                                                                                                                           
-        echo "group already exists"                                                                                                                                                                                                
-    fi                                                                                                                                                                                                                             
-
-    echo "adding $OSPARC_USER to group $CONTAINER_GROUPNAME..."                                                                                                                                                                        
-    usermod --append --groups "$CONTAINER_GROUPNAME" "$OSPARC_USER"                                                                                                                                                                    
-
-    echo "changing owner ship of state directory /home/${OSPARC_USER}/work/workspace"                                                                                                                                                 
-    chown --recursive "$OSPARC_USER" "/home/${OSPARC_USER}/work/workspace"                                                                                                                                                                 
-    echo "changing owner ship of state directory ${DY_SIDECAR_PATH_INPUTS}"                                                                                                                                                       
-    chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_INPUTS}"                                                                                                                                                                       
-    echo "changing owner ship of state directory ${DY_SIDECAR_PATH_OUTPUTS}"                                                                                                                                                      
-    chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_OUTPUTS}"                                                                                                                                                                      
-fi                    
-
-exec gosu "$OSPARC_USER" /docker/map.bash
+if [ "$HOST_USERID" -eq 0 ]; then
+	echo "Warning: Folder mounted owned by root user... adding $OSPARC_USER to root..."
+	addgroup "$OSPARC_USER" root
+else
+	echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONTAINER_GROUPNAME'..."
+	# take host's credentials in $OSPARC_USER
+	if [ -z "$CONTAINER_GROUPNAME" ]; then
+		echo "Creating new group my$OSPARC_USER"
+		CONTAINER_GROUPNAME=my$OSPARC_USER
+		addgroup --gid "$HOST_GROUPID" "$CONTAINER_GROUPNAME"
+	else
+		echo "group already exists"
+	fi
+
+	echo "adding $OSPARC_USER to group $CONTAINER_GROUPNAME..."
+	usermod --append --groups "$CONTAINER_GROUPNAME" "$OSPARC_USER"
+
+	echo "changing owner ship of state directory /home/${OSPARC_USER}/work/workspace"
+	chown --recursive "$OSPARC_USER" "/home/${OSPARC_USER}/work/workspace"
+	echo "changing owner ship of state directory ${DY_SIDECAR_PATH_INPUTS}"
+	chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_INPUTS}"
+	echo "changing owner ship of state directory ${DY_SIDECAR_PATH_OUTPUTS}"
+	chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_OUTPUTS}"
+fi
+
+exec gosu "$OSPARC_USER" /docker/main.bash

docker_scripts/map.bash → docker_scripts/main.bash

@@ -2,5 +2,5 @@
 
 pip install -r /docker/requirements.txt
 echo "Starting map python code"
-python3 /docker/map.py
+python3 /docker/main.py
 echo "Closing map python code"

docker_scripts/main.py

@@ -0,0 +1,74 @@
+import http.server
+import logging
+import pathlib as pl
+import socketserver
+import threading
+import time
+import typing
+
+import pydantic as pyda
+import pydantic_settings
+
+import parallelrunner
+
+logging.basicConfig(
+    level=logging.INFO, format="[%(filename)s:%(lineno)d] %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+HTTP_PORT = 8888
+INPUT_CONF_KEY = "input_3"
+
+
+def main():
+    """Main"""
+
+    settings = MainSettings()
+    config_path = settings.input_path / INPUT_CONF_KEY / "parallelrunner.json"
+
+    waiter = 0
+    while not config_path.exists():
+        if waiter % 10 == 0:
+            logger.info("Waiting for parallelrunner.json to exist ...")
+        time.sleep(settings.file_polling_interval)
+        waiter += 1
+
+    settings = settings.parse_file(config_path)
+    logging.info(f"Received the following settings: {settings}")
+
+    http_dir_path = pl.Path(__file__).parent / "http"
+
+    class HTTPHandler(http.server.SimpleHTTPRequestHandler):
+        def __init__(self, *args, **kwargs):
+            super().__init__(
+                *args, **kwargs, directory=http_dir_path.resolve()
+            )
+
+    maprunner = parallelrunner.ParallelRunner(**settings.dict())
+
+    try:
+        logger.info(
+            f"Starting http server at port {HTTP_PORT} and serving path {http_dir_path}"
+        )
+        with socketserver.TCPServer(("", HTTP_PORT), HTTPHandler) as httpd:
+            httpd_thread = threading.Thread(target=httpd.serve_forever)
+            httpd_thread.start()
+            maprunner.setup()
+            maprunner.start()
+            maprunner.teardown()
+            httpd.shutdown()
+    except Exception as err:  # pylint: disable=broad-except
+        logger.error(f"{err} . Stopping %s", exc_info=True)
+
+
+class MainSettings(pydantic_settings.BaseSettings):
+    batch_mode: bool = False
+    file_polling_interval: int = 1
+    input_path: pyda.DirectoryPath = pyda.Field(alias="DY_SIDECAR_PATH_INPUTS")
+    output_path: pyda.DirectoryPath = pyda.Field(
+        alias="DY_SIDECAR_PATH_OUTPUTS"
+    )
+
+
+if __name__ == "__main__":
+    main()