diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 84c3170..d58bda1 100755 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.0 +current_version = 0.2.1 commit = False message = service version: {current_version} → {new_version} tag = False diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml old mode 100644 new mode 100755 diff --git a/.osparc/osparc-meta-dakota/metadata.yml b/.osparc/osparc-meta-dakota/metadata.yml index 8383fe3..9f5ef30 100755 --- a/.osparc/osparc-meta-dakota/metadata.yml +++ b/.osparc/osparc-meta-dakota/metadata.yml @@ -1,8 +1,8 @@ name: DakotaService description: "DakotaServiceService" key: simcore/services/dynamic/osparc-meta-dakota -version: 0.1.0 -integration-version: 0.1.0 +version: 0.2.1 +integration-version: 0.2.1 type: dynamic authors: - name: Werner Van Geit @@ -22,6 +22,12 @@ inputs: description: Map feedback channel type: data:*/* + input_2: + displayOrder: 2.0 + label: Dakota service settings + description: + Dakota service settings file + type: data:*/* outputs: output_0: displayOrder: 0.0 @@ -33,6 +39,11 @@ outputs: label: Map commands description: Map command channel type: data:*/* + conf_json_schema: + displayOrder: 2.0 + label: JSON schema + description: JSON schema of configuration file + type: data:*/* boot-options: boot_mode: label: Boot mode diff --git a/Dockerfile b/Dockerfile index 8c2fcc5..e1d9d5f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -19,10 +19,10 @@ RUN pip3 install itis-dakota osparc_filecomms --upgrade USER osparcuser WORKDIR /home/osparcuser +RUN python3 -m venv venv +RUN . ./venv/bin/activate && pip3 install --upgrade -r /docker/requirements.txt USER root - EXPOSE 8888 ENTRYPOINT [ "/bin/bash", "-c", "/docker/entrypoint.bash" ] -CMD [ "/bin/bash", "-c", "/docker/runner.bash "] diff --git a/Makefile b/Makefile index ca1de19..5777868 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL = /bin/sh MAKEFLAGS += -j3 export DOCKER_IMAGE_NAME ?= osparc-meta-dakota -export DOCKER_IMAGE_TAG ?= 0.1.0 +export DOCKER_IMAGE_TAG ?= 0.2.1 export MASTER_AWS_REGISTRY ?= registry.osparc-master-zmt.click export MASTER_REGISTRY ?= registry.osparc-master.speag.com @@ -16,7 +16,7 @@ define _bumpversion # upgrades as $(subst $(1),,$@) version, commits and tags @docker run -it --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ -u $(shell id -u):$(shell id -g) \ - itisfoundation/ci-service-integration-library:v1.0.1-dev-33 \ + itisfoundation/ci-service-integration-library:latest \ sh -c "cd /${DOCKER_IMAGE_NAME} && bump2version --verbose --list --config-file $(1) $(subst $(2),,$@)" endef @@ -31,7 +31,7 @@ version-patch version-minor version-major: .bumpversion.cfg ## increases service compose-spec: ## runs ooil to assemble the docker-compose.yml file @docker run --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ -u $(shell id -u):$(shell id -g) \ - itisfoundation/ci-service-integration-library:v1.0.4 \ + itisfoundation/ci-service-integration-library:latest \ sh -c "cd /${DOCKER_IMAGE_NAME} && ooil compose" clean: clean-validation @@ -39,12 +39,13 @@ clean: clean-validation .PHONY: build build: clean compose-spec ## build docker image + chmod -R 755 docker_scripts docker compose build clean-validation: rm -rf validation-tmp cp -r validation validation-tmp - chmod -R 770 validation-tmp + chmod -R 775 validation-tmp run-compose-local: docker compose down @@ -52,6 +53,7 @@ run-compose-local: run-mock-mapservice: pip install osparc-filecomms + sleep 5 MOCK_MAP_INPUT_PATH=validation-tmp/outputs/output_1 MOCK_MAP_OUTPUT_PATH=validation-tmp/inputs/input_1 python validation-client/mock_mapservice.py run-validation-client: diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/docker-compose-local.yml b/docker-compose-local.yml index e35e41a..2f4c9cf 100755 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -1,7 +1,6 @@ -version: '3.7' services: osparc-meta-dakota: - image: simcore/services/dynamic/osparc-meta-dakota:0.1.0 + image: simcore/services/dynamic/osparc-meta-dakota:0.2.1 ports: - "8888:8888" environment: diff --git a/docker_scripts/dakota.bash b/docker_scripts/dakota.bash deleted file mode 100755 index 8c9a22b..0000000 --- a/docker_scripts/dakota.bash +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -pip install -r /docker/requirements.txt -python3 /docker/dakota-start.py diff --git a/docker_scripts/dakota-start.py b/docker_scripts/dakota_start.py similarity index 56% rename from docker_scripts/dakota-start.py rename to docker_scripts/dakota_start.py index 75555c7..a3953f5 100755 --- a/docker_scripts/dakota-start.py +++ b/docker_scripts/dakota_start.py @@ -1,12 +1,10 @@ import contextlib -import http.server import logging +import multiprocessing import os import pathlib as pl import shutil -import socketserver import sys -import threading import time import uuid @@ -24,53 +22,21 @@ f"{str(pl.Path(__file__).resolve().parent)}" ) -import tools.maps # NOQA - - -NOISE_MUS = [0.0, 0.0] -NOISE_SIGMAS = [5.0, 10.0] - -POLLING_TIME = 0.1 -HTTP_PORT = 8888 - - -def main(): - dakota_service = DakotaService() - - http_dir_path = pl.Path(__file__).parent / "http" - - class HTTPHandler(http.server.SimpleHTTPRequestHandler): - def __init__(self, *args, **kwargs): - super().__init__( - *args, **kwargs, directory=http_dir_path.resolve() - ) - - try: - logger.info( - f"Starting http server at port {HTTP_PORT} and serving path {http_dir_path}" - ) - with socketserver.TCPServer(("", HTTP_PORT), HTTPHandler) as httpd: - httpd_thread = threading.Thread(target=httpd.serve_forever) - httpd_thread.start() - dakota_service.start() - httpd.shutdown() - except Exception as err: # pylint: disable=broad-except - logger.error(f"{err} . Stopping %s", exc_info=True) +import map.maps # NOQA class DakotaService: - def __init__(self): + def __init__(self, settings): + self.settings = settings self.uuid = uuid.uuid4() self.caller_uuid = None self.map_uuid = None - self.input_dir_path = pl.Path(os.environ["DY_SIDECAR_PATH_INPUTS"]) - self.input0_dir_path = self.input_dir_path / "input_0" - self.input1_dir_path = self.input_dir_path / "input_1" + self.input0_dir_path = self.settings.input_path / "input_0" + self.input1_dir_path = self.settings.input_path / "input_1" - self.output_dir_path = pl.Path(os.environ["DY_SIDECAR_PATH_OUTPUTS"]) - self.output0_dir_path = self.output_dir_path / "output_0" - self.output1_dir_path = self.output_dir_path / "output_1" + self.output0_dir_path = self.settings.output_path / "output_0" + self.output1_dir_path = self.settings.output_path / "output_1" self.dakota_conf_path = self.input0_dir_path / "dakota.in" @@ -95,7 +61,7 @@ def clean_output(self, dir_path): item.unlink() def start(self): - self.map_object = tools.maps.oSparcFileMap( + self.map_object = map.maps.oSparcFileMap( self.map_reply_file_path.resolve(), self.map_caller_file_path.resolve(), ) @@ -103,7 +69,7 @@ def start(self): self.caller_uuid = self.caller_handshaker.shake() while not self.dakota_conf_path.exists(): - time.sleep(POLLING_TIME) + time.sleep(self.settings.file_polling_interval) dakota_conf = self.dakota_conf_path.read_text() clear_directory( @@ -117,7 +83,62 @@ def start(self): dirs_exist_ok=True, ) - self.start_dakota(dakota_conf, self.output0_dir_path) + first_error_time = None + + while True: + try: + process = multiprocessing.Process( + target=self.start_dakota, + args=(dakota_conf, self.output0_dir_path), + ) + process.start() + process.join() + logging.info(f"PROCESS ended with exitcode {process.exitcode}") + if process.exitcode != 0: + raise RuntimeError("Dakota subprocess failed") + break + except RuntimeError as error: + if not self.settings.restart_on_error: + raise error + if first_error_time is None: + first_error_time = time.time() + if ( + time.time() - first_error_time + >= self.settings.restart_on_error_max_time + ): + logging.info( + "Received a RunTimeError from Dakota, " + "max retry time reached, raising error" + ) + raise error + else: + logging.info( + f"Received a RunTimeError from Dakota ({error}), " + "retrying ..." + ) + time.sleep(self.settings.restart_on_error_polling_interval) + max_wait_time = self.settings.restart_on_error_max_time - ( + time.time() - first_error_time + ) + logging.info( + f"Will wait for a maximum of {max_wait_time} " + "seconds for a change in dakota conf file..." + ) + dakota_conf = self.wait_for_dakota_conf_change( + dakota_conf, max_wait_time + ) + logging.info("Change in dakota conf file detected") + continue + + def wait_for_dakota_conf_change(self, old_dakota_conf, max_wait_time): + new_dakota_conf = None + start_time = time.time() + while new_dakota_conf is None or new_dakota_conf == old_dakota_conf: + if time.time() - start_time > max_wait_time: + raise TimeoutError("Waiting too long for new dakota.conf") + new_dakota_conf = self.dakota_conf_path.read_text() + time.sleep(self.settings.file_polling_interval) + return new_dakota_conf def model_callback(self, dak_inputs): param_sets = [ @@ -184,7 +205,3 @@ def clear_directory(path): os.unlink(item_path) elif os.path.isdir(item_path): shutil.rmtree(item_path) - - -if __name__ == "__main__": - main() diff --git a/docker_scripts/entrypoint.bash b/docker_scripts/entrypoint.bash index bf23bb8..335874a 100755 --- a/docker_scripts/entrypoint.bash +++ b/docker_scripts/entrypoint.bash @@ -1,10 +1,11 @@ #!/bin/bash set -euo pipefail + IFS=$'\n\t' INFO="INFO: [$(basename "$0")] " -echo "$INFO" "Starting container for map ..." +echo "$INFO" "Starting container for dakotarunner ..." HOST_USERID=$(stat -c %u "${DY_SIDECAR_PATH_INPUTS}") HOST_GROUPID=$(stat -c %g "${DY_SIDECAR_PATH_INPUTS}") @@ -13,28 +14,28 @@ CONTAINER_GROUPNAME=$(getent group | grep "${HOST_GROUPID}" | cut --delimiter=: OSPARC_USER='osparcuser' if [ "$HOST_USERID" -eq 0 ]; then - echo "Warning: Folder mounted owned by root user... adding $OSPARC_USER to root..." - addgroup "$OSPARC_USER" root + # echo "Warning: Folder mounted owned by root user... adding $OSPARC_USER to root..." + addgroup "$OSPARC_USER" root else - echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONTAINER_GROUPNAME'..." - # take host's credentials in $OSPARC_USER - if [ -z "$CONTAINER_GROUPNAME" ]; then - echo "Creating new group my$OSPARC_USER" - CONTAINER_GROUPNAME=my$OSPARC_USER - addgroup --gid "$HOST_GROUPID" "$CONTAINER_GROUPNAME" - else - echo "group already exists" - fi - - echo "adding $OSPARC_USER to group $CONTAINER_GROUPNAME..." - usermod --append --groups "$CONTAINER_GROUPNAME" "$OSPARC_USER" - - echo "changing owner ship of state directory /home/${OSPARC_USER}/work/workspace" - chown --recursive "$OSPARC_USER" "/home/${OSPARC_USER}/work/workspace" - echo "changing owner ship of state directory ${DY_SIDECAR_PATH_INPUTS}" - chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_INPUTS}" - echo "changing owner ship of state directory ${DY_SIDECAR_PATH_OUTPUTS}" - chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_OUTPUTS}" + # echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONTAINER_GROUPNAME'..." + # take host's credentials in $OSPARC_USER + if [ -z "$CONTAINER_GROUPNAME" ]; then + # echo "Creating new group my$OSPARC_USER" + CONTAINER_GROUPNAME=my$OSPARC_USER + addgroup --gid "$HOST_GROUPID" "$CONTAINER_GROUPNAME" + else + echo "group already exists" + fi + + # echo "adding $OSPARC_USER to group $CONTAINER_GROUPNAME..." + usermod --append --groups "$CONTAINER_GROUPNAME" "$OSPARC_USER" + + # echo "changing owner ship of state directory /home/${OSPARC_USER}/work/workspace" + chown --recursive "$OSPARC_USER" "/home/${OSPARC_USER}/work/workspace" + # echo "changing owner ship of state directory ${DY_SIDECAR_PATH_INPUTS}" + chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_INPUTS}" + # echo "changing owner ship of state directory ${DY_SIDECAR_PATH_OUTPUTS}" + chown --recursive "$OSPARC_USER" "${DY_SIDECAR_PATH_OUTPUTS}" fi -exec gosu "$OSPARC_USER" /docker/dakota.bash +exec gosu "$OSPARC_USER" /docker/main.bash diff --git a/docker_scripts/http/server.py b/docker_scripts/http/server.py new file mode 100755 index 0000000..e8d188d --- /dev/null +++ b/docker_scripts/http/server.py @@ -0,0 +1,37 @@ +import http.server +import logging +import pathlib as pl +import socketserver +import threading + +logging.basicConfig( + level=logging.INFO, format="[%(filename)s:%(lineno)d] %(message)s" +) +logger = logging.getLogger(__name__) + +HTTP_PORT = 8888 + + +def main(): + http_dir_path = pl.Path(__file__).parent + + class HTTPHandler(http.server.SimpleHTTPRequestHandler): + def __init__(self, *args, **kwargs): + super().__init__( + *args, **kwargs, directory=http_dir_path.resolve() + ) + + try: + logger.info( + f"Starting http server at port {HTTP_PORT} and serving path {http_dir_path}" + ) + with socketserver.TCPServer(("", HTTP_PORT), HTTPHandler) as httpd: + httpd_thread = threading.Thread(target=httpd.serve_forever) + httpd_thread.start() + httpd.serve_forever() + except Exception as err: # pylint: disable=broad-except + logger.error(f"{err} . Stopping %s", exc_info=True) + + +if __name__ == "__main__": + main() diff --git a/docker_scripts/main.bash b/docker_scripts/main.bash new file mode 100755 index 0000000..74d57a8 --- /dev/null +++ b/docker_scripts/main.bash @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +cd /docker/http +python server.py & + +cd ${HOME} +source ./venv/bin/activate +python3 /docker/main.py diff --git a/docker_scripts/main.py b/docker_scripts/main.py new file mode 100755 index 0000000..bcadf9b --- /dev/null +++ b/docker_scripts/main.py @@ -0,0 +1,95 @@ +import json +import logging + +import pydantic as pyda +import pydantic_settings + +import dakota_start +import tools + +logging.basicConfig( + level=logging.INFO, format="[%(filename)s:%(lineno)d] %(message)s" +) +logger = logging.getLogger(__name__) + +INPUT_CONF_KEY = "input_2" +CONF_SCHEMA_KEY = "conf_json_schema" + +DEFAULT_FILE_POLLING_INTERVAL = 0.1 +RESTART_ON_ERROR_MAX_TIME = 10.0 +RESTART_ON_ERROR_POLLING_INTERVAL = 1.0 + + +def main(): + """Main""" + + settings = DakotaDynamicSettings() + + # Wait for and read the settings file + logger.info( + f"Waiting for settings file to appear at {settings.settings_file_path}" + ) + settings.read_settings_file() + logger.info("Settings file was read") + + # Create and start the dakota service + dakservice = dakota_start.DakotaService(settings) + dakservice.start() + +class DakotaDynamicSettings: + def __init__(self): + self._settings = self.DakotaMainSettings() + conf_json_schema_path = ( + self._settings.output_path / CONF_SCHEMA_KEY / "schema.json" + ) + + settings_schema = self._settings.model_json_schema() + + # Hide some settings from the user + for field_name in [ + "DY_SIDECAR_PATH_INPUTS", + "DY_SIDECAR_PATH_OUTPUTS", + ]: + settings_schema["properties"].pop(field_name) + + conf_json_schema_path.write_text(json.dumps(settings_schema, indent=2)) + + self.settings_file_path = ( + self._settings.input_path / INPUT_CONF_KEY / "settings.json" + ) + + def __getattr__(self, name): + if name in self.__dict__: + return self.__dict__[name] + else: + self.read_settings_file() + return getattr(self._settings, name) + + def read_settings_file(self): + tools.wait_for_path(self.settings_file_path) + self._settings = self._settings.parse_file(self.settings_file_path) + + class DakotaMainSettings(pydantic_settings.BaseSettings): + batch_mode: bool = pyda.Field(default=False) + file_polling_interval: float = pyda.Field( + default=DEFAULT_FILE_POLLING_INTERVAL + ) + input_path: pyda.DirectoryPath = pyda.Field( + alias="DY_SIDECAR_PATH_INPUTS" + ) + output_path: pyda.DirectoryPath = pyda.Field( + alias="DY_SIDECAR_PATH_OUTPUTS" + ) + restart_on_error: bool = pyda.Field( + default=False + ) + restart_on_error_max_time: float = pyda.Field( + default=RESTART_ON_ERROR_MAX_TIME, ge=0 + ) + restart_on_error_polling_interval: float = pyda.Field( + default=RESTART_ON_ERROR_POLLING_INTERVAL, ge=0 + ) + + +if __name__ == "__main__": + main() diff --git a/docker_scripts/main.sh b/docker_scripts/main.sh deleted file mode 100755 index 31a1924..0000000 --- a/docker_scripts/main.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set -o errexit -set -o nounset - -echo "Starting main.sh" -echo "Creating virtual environment" -python3 -m venv --system-site-packages --symlinks --upgrade ${OSPARC_VENV_DIR} -${OSPARC_VENV_DIR}/bin/pip install -qU pip wheel setuptools - - -[ ! -z "${OSPARC_REQUIREMENTS_TXT}" ] && ${OSPARC_VENV_DIR}/bin/pip install -qr ${OSPARC_REQUIREMENTS_TXT} - -echo "Executing code ${OSPARC_USER_ENTRYPOINT_PATH}" -cd ${OSPARC_USER_ENTRYPOINT_DIR} -${OSPARC_VENV_DIR}/bin/python3 ${OSPARC_USER_ENTRYPOINT_PATH} -echo "Stopping main.sh" diff --git a/docker_scripts/tools/maps.py b/docker_scripts/map/maps.py similarity index 100% rename from docker_scripts/tools/maps.py rename to docker_scripts/map/maps.py diff --git a/docker_scripts/osparc-0.6.5.post0-py3-none-any.whl b/docker_scripts/osparc-0.6.5.post0-py3-none-any.whl deleted file mode 100755 index e76939e..0000000 Binary files a/docker_scripts/osparc-0.6.5.post0-py3-none-any.whl and /dev/null differ diff --git a/docker_scripts/osparc_client-0.6.5.post0-py3-none-any.whl b/docker_scripts/osparc_client-0.6.5.post0-py3-none-any.whl deleted file mode 100755 index 7217441..0000000 Binary files a/docker_scripts/osparc_client-0.6.5.post0-py3-none-any.whl and /dev/null differ diff --git a/docker_scripts/requirements.txt b/docker_scripts/requirements.txt index e69de29..39cf0af 100755 --- a/docker_scripts/requirements.txt +++ b/docker_scripts/requirements.txt @@ -0,0 +1,5 @@ +itis-dakota +osparc-filecomms +pydantic +pydantic-settings==2.5.2 +watchdog diff --git a/docker_scripts/tools.py b/docker_scripts/tools.py new file mode 100755 index 0000000..00b9a67 --- /dev/null +++ b/docker_scripts/tools.py @@ -0,0 +1,70 @@ +import json +import logging +import pathlib as pl +import time + +from watchdog.events import FileSystemEventHandler +from watchdog.observers import Observer + +DEFAULT_FILE_POLLING_INTERVAL = 5 + +logging.basicConfig( + level=logging.INFO, format="[%(filename)s:%(lineno)d] %(message)s" +) +logger = logging.getLogger(__name__) + + +def wait_for_path(file_path): + path = pl.Path(file_path).resolve() + if path.exists(): + return str(path) + + # Find the closest existing parent directory + watch_dir = path + while not watch_dir.exists(): + watch_dir = watch_dir.parent + + class Handler(FileSystemEventHandler): + def __init__(self): + self.created = False + + def on_created(self, event): + nonlocal path + created_path = pl.Path(event.src_path).resolve() + if created_path == path: + self.created = True + elif path.is_relative_to(created_path): + # Update path if a parent directory was created + path = created_path / path.relative_to(created_path) + + handler = Handler() + observer = Observer() + observer.schedule(handler, str(watch_dir), recursive=True) + observer.start() + + try: + while not handler.created: + if path.exists(): + return str(path) + observer.join(0.1) + return str(path) + finally: + observer.stop() + observer.join() + + +def load_json( + file_path, wait=True, file_polling_interval=DEFAULT_FILE_POLLING_INTERVAL +): + if wait: + wait_for_path(file_path) + + while True: + try: + content = json.loads(file_path.read_text()) + break + except json.decoder.JSONDecodeError: + logger.info(f"JSON read error, retrying read from {file_path}") + time.sleep(file_polling_interval) + + return content diff --git a/validation-client/client.py b/validation-client/client.py old mode 100644 new mode 100755 index 2d3dbe6..6419d24 --- a/validation-client/client.py +++ b/validation-client/client.py @@ -54,8 +54,15 @@ def main(): dakota_in_template = string.Template(dakota_in_template_path.read_text()) dakota_in_path = client_output_path / "dakota.in" + dakota_in = dakota_in_template.substitute(model=model_string) + dakota_in2 = dakota_in_template.substitute(model=model_string) + + dakota_in_path.write_text( + dakota_in.replace("environment", "wrong") + ) + time.sleep(5) dakota_in_path.write_text( - dakota_in_template.substitute(model=model_string) + dakota_in2 ) while not os.path.exists(dak_opt_path): diff --git a/validation-client/dakota.in.template b/validation-client/dakota.in.template old mode 100644 new mode 100755 diff --git a/validation-client/dakota.rst b/validation-client/dakota.rst old mode 100644 new mode 100755 diff --git a/validation-client/mock_mapservice.py b/validation-client/mock_mapservice.py old mode 100644 new mode 100755 diff --git a/validation/inputs/input_0/.gitignore b/validation/inputs/input_0/.gitignore old mode 100644 new mode 100755 diff --git a/validation/inputs/input_1/.gitignore b/validation/inputs/input_1/.gitignore old mode 100644 new mode 100755 diff --git a/validation/inputs/input_2/settings.json b/validation/inputs/input_2/settings.json new file mode 100644 index 0000000..298a1bf --- /dev/null +++ b/validation/inputs/input_2/settings.json @@ -0,0 +1,3 @@ +{ + "restart_on_error": true +} diff --git a/validation/opt.dat.expected b/validation/opt.dat.expected old mode 100644 new mode 100755 diff --git a/validation/outputs/conf_json_schema/.gitignore b/validation/outputs/conf_json_schema/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/validation/outputs/output_0/.gitignore b/validation/outputs/output_0/.gitignore old mode 100644 new mode 100755