From 45439989054cf2d6caf7f3119205e260e53f44b1 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 23 Jul 2024 16:03:52 +0200 Subject: [PATCH 1/5] Write json schema to output --- .osparc/osparc-meta-parallelrunner/metadata.yml | 5 +++++ Makefile | 2 +- docker_scripts/main.py | 14 +++++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.osparc/osparc-meta-parallelrunner/metadata.yml b/.osparc/osparc-meta-parallelrunner/metadata.yml index bb69124..daea72f 100755 --- a/.osparc/osparc-meta-parallelrunner/metadata.yml +++ b/.osparc/osparc-meta-parallelrunner/metadata.yml @@ -40,6 +40,11 @@ outputs: label: Output values description: Output files uploaded from the outputs folder type: data:*/* + conf_json_schema: + displayOrder: 2.0 + label: JSON schema + description: JSON schema of configuration file + type: data:*/* boot-options: boot_mode: label: Boot mode diff --git a/Makefile b/Makefile index 4157afb..8d49e42 100755 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ build: clean compose-spec ## build docker image docker compose build validation-clean: - sudo rm -rf validation-tmp + rm -rf validation-tmp cp -r validation validation-tmp chmod -R 770 validation-tmp diff --git a/docker_scripts/main.py b/docker_scripts/main.py index 41344b5..547c27d 100755 --- a/docker_scripts/main.py +++ b/docker_scripts/main.py @@ -1,10 +1,10 @@ import http.server +import json import logging import pathlib as pl import socketserver import threading import time -import typing import pydantic as pyda import pydantic_settings @@ -18,6 +18,7 @@ HTTP_PORT = 8888 INPUT_CONF_KEY = "input_3" +CONF_SCHEMA_KEY = "conf_json_schema" FILE_POLLING_INTERVAL = 1 # second @@ -29,7 +30,14 @@ def main(): """Main""" - settings = MainSettings() + settings = ParallelRunnerMainSettings() + settings_schema = settings.model_json_schema() + logger.info(settings_schema) + conf_json_schema_path = ( + settings.output_path / CONF_SCHEMA_KEY / "schema.json" + ) + conf_json_schema_path.write_text(json.dumps(settings_schema, indent=2)) + config_path = settings.input_path / INPUT_CONF_KEY / "parallelrunner.json" http_dir_path = pl.Path(__file__).parent / "http" @@ -71,7 +79,7 @@ def __init__(self, *args, **kwargs): logger.error(f"{err} . Stopping %s", exc_info=True) -class MainSettings(pydantic_settings.BaseSettings): +class ParallelRunnerMainSettings(pydantic_settings.BaseSettings): batch_mode: bool = False file_polling_interval: int = FILE_POLLING_INTERVAL input_path: pyda.DirectoryPath = pyda.Field(alias="DY_SIDECAR_PATH_INPUTS") From 12b9121da103a6a6788d5e4c20b20bb0764ff6f0 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 23 Jul 2024 16:04:17 +0200 Subject: [PATCH 2/5] Bump version --- .bumpversion.cfg | 2 +- .osparc/osparc-meta-parallelrunner/metadata.yml | 2 +- Makefile | 2 +- docker-compose-local.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9e5579c..945ed3c 100755 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.1.20 +current_version = 0.2.0 commit = False message = service version: {current_version} → {new_version} tag = False diff --git a/.osparc/osparc-meta-parallelrunner/metadata.yml b/.osparc/osparc-meta-parallelrunner/metadata.yml index daea72f..0a8ad54 100755 --- a/.osparc/osparc-meta-parallelrunner/metadata.yml +++ b/.osparc/osparc-meta-parallelrunner/metadata.yml @@ -1,7 +1,7 @@ name: ParallelRunner description: "ParallelRunnerService" key: simcore/services/dynamic/osparc-meta-parallelrunner -version: 0.1.20 +version: 0.2.0 integration-version: 2.0.0 type: dynamic authors: diff --git a/Makefile b/Makefile index 8d49e42..bd1730a 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL = /bin/sh MAKEFLAGS += -j2 export DOCKER_IMAGE_NAME ?= osparc-meta-parallelrunner -export DOCKER_IMAGE_TAG ?= 0.1.20 +export DOCKER_IMAGE_TAG ?= 0.2.0 export MASTER_AWS_REGISTRY ?= registry.osparc-master-zmt.click export MASTER_REGISTRY ?= registry.osparc-master.speag.com diff --git a/docker-compose-local.yml b/docker-compose-local.yml index a4bf576..3326bdd 100755 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -1,6 +1,6 @@ services: osparc-meta-parallelrunner: - image: simcore/services/dynamic/osparc-meta-parallelrunner:0.1.20 + image: simcore/services/dynamic/osparc-meta-parallelrunner:0.2.0 ports: - "8888:8888" environment: From 455079c08464425d2ff5559fb4323b234de34c7f Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 30 Jul 2024 16:48:21 +0200 Subject: [PATCH 3/5] Add mechanism to specify job timeouts --- docker_scripts/main.py | 3 +++ docker_scripts/parallelrunner.py | 15 +++++++++++++-- validation/inputs/input_3/parallelrunner.json | 3 ++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/docker_scripts/main.py b/docker_scripts/main.py index 547c27d..a44a61a 100755 --- a/docker_scripts/main.py +++ b/docker_scripts/main.py @@ -26,6 +26,8 @@ JOB_CREATE_ATTEMPTS_DELAY = 5 MAX_JOB_TRIALS = 5 +JOB_TIMEOUT = None + def main(): """Main""" @@ -90,6 +92,7 @@ class ParallelRunnerMainSettings(pydantic_settings.BaseSettings): file_polling_interval: int = FILE_POLLING_INTERVAL max_job_create_attempts: int = MAX_JOB_CREATE_ATTEMPTS job_create_attempts_delay: int = JOB_CREATE_ATTEMPTS_DELAY + job_timeout: None | float = JOB_TIMEOUT if __name__ == "__main__": diff --git a/docker_scripts/parallelrunner.py b/docker_scripts/parallelrunner.py index 34ce12b..d542179 100755 --- a/docker_scripts/parallelrunner.py +++ b/docker_scripts/parallelrunner.py @@ -39,6 +39,7 @@ def __init__( max_job_trials=None, max_job_create_attempts=None, job_create_attempts_delay=None, + job_timeout=None, ): """Constructor""" self.test_mode = False @@ -48,6 +49,7 @@ def __init__( self.max_job_trials = max_job_trials self.max_job_create_attempts = max_job_create_attempts self.job_create_attempts_delay = job_create_attempts_delay + self.job_timeout = job_timeout self.input_path = input_path # path where osparc write all our input self.output_path = output_path # path where osparc write all our input @@ -291,13 +293,13 @@ def run_job(self, job_inputs, input_batch): """Run a job with given inputs""" logger.debug(f"Sending inputs: {job_inputs}") - if self.test_mode: logger.info("Map in test mode, just returning input") done_batch = self.process_job_outputs( job_inputs, input_batch, "SUCCESS" ) + time.sleep(1) return done_batch @@ -443,7 +445,16 @@ def map_func(batch, trial_number=1): job_inputs = self.create_job_inputs(task_input) - output_batch = self.run_job(job_inputs, batch) + with pathos.pools.ThreadPool(nodes=1) as timeout_pool: + output_batch_waiter = timeout_pool.apipe( + self.run_job, job_inputs, batch + ) + output_batch = output_batch_waiter.get( + timeout=self.job_timeout + ) + timeout_pool.close() + timeout_pool.join() + timeout_pool.clear() # Pool is singleton, need to clear old pool self.n_of_finished_batches += 1 logger.info( diff --git a/validation/inputs/input_3/parallelrunner.json b/validation/inputs/input_3/parallelrunner.json index 619408a..e775459 100644 --- a/validation/inputs/input_3/parallelrunner.json +++ b/validation/inputs/input_3/parallelrunner.json @@ -1,3 +1,4 @@ { - "batch_mode": true + "batch_mode": true, + "job_timeout": 60 } From dc47ea212f5d17b975b9f59c22233f8661f2f2d8 Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 30 Jul 2024 16:48:56 +0200 Subject: [PATCH 4/5] Bump version --- .bumpversion.cfg | 2 +- .osparc/osparc-meta-parallelrunner/metadata.yml | 2 +- Makefile | 2 +- docker-compose-local.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 945ed3c..92d9fd4 100755 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.0 +current_version = 0.2.1 commit = False message = service version: {current_version} → {new_version} tag = False diff --git a/.osparc/osparc-meta-parallelrunner/metadata.yml b/.osparc/osparc-meta-parallelrunner/metadata.yml index 0a8ad54..073d576 100755 --- a/.osparc/osparc-meta-parallelrunner/metadata.yml +++ b/.osparc/osparc-meta-parallelrunner/metadata.yml @@ -1,7 +1,7 @@ name: ParallelRunner description: "ParallelRunnerService" key: simcore/services/dynamic/osparc-meta-parallelrunner -version: 0.2.0 +version: 0.2.1 integration-version: 2.0.0 type: dynamic authors: diff --git a/Makefile b/Makefile index bd1730a..7e412fe 100755 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL = /bin/sh MAKEFLAGS += -j2 export DOCKER_IMAGE_NAME ?= osparc-meta-parallelrunner -export DOCKER_IMAGE_TAG ?= 0.2.0 +export DOCKER_IMAGE_TAG ?= 0.2.1 export MASTER_AWS_REGISTRY ?= registry.osparc-master-zmt.click export MASTER_REGISTRY ?= registry.osparc-master.speag.com diff --git a/docker-compose-local.yml b/docker-compose-local.yml index 3326bdd..64e005f 100755 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -1,6 +1,6 @@ services: osparc-meta-parallelrunner: - image: simcore/services/dynamic/osparc-meta-parallelrunner:0.2.0 + image: simcore/services/dynamic/osparc-meta-parallelrunner:0.2.1 ports: - "8888:8888" environment: From 68be5049d205de506d37bc40eb4ac48e9373f85e Mon Sep 17 00:00:00 2001 From: Werner Van Geit Date: Tue, 30 Jul 2024 16:55:49 +0200 Subject: [PATCH 5/5] Add missing validation dir --- validation/outputs/conf_json_schema/.gitignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 validation/outputs/conf_json_schema/.gitignore diff --git a/validation/outputs/conf_json_schema/.gitignore b/validation/outputs/conf_json_schema/.gitignore new file mode 100644 index 0000000..e69de29