dy-static-file-server will consider initial input state as to be synced

services/dy-static-file-server/.cookiecutterrc

@@ -35,4 +35,4 @@ default_context:
     project_slug:              'dy-static-file-server'
     project_type:              'computational'
     release_date:              '2021'
-    version:                   '2.0.6'
+    version:                   '2.0.7'

services/dy-static-file-server/Makefile

@@ -107,6 +107,7 @@ info-build: ## displays info on the built image
 tests-unit tests-integration: ## runs integration and unit tests
 	@.venv/bin/pytest -vv \
 		--basetemp=$(CURDIR)/tmp \
+		-vv -s --log-cli-level=DEBUG \
 		--exitfirst \
 		--failed-first \
 		--pdb \

services/dy-static-file-server/VERSION

@@ -1 +1 @@
-2.0.6
+2.0.7

services/dy-static-file-server/docker-compose-meta.yml

@@ -11,7 +11,7 @@ services:
         io.simcore.name: '{"name": "dy-static-file-server"}'
         io.simcore.outputs: '{"outputs": {}}'
         io.simcore.type: '{"type": "dynamic"}'
-        io.simcore.version: '{"version": "2.0.6"}'
+        io.simcore.version: '{"version": "2.0.7"}'
         org.label-schema.build-date: ${BUILD_DATE}
         org.label-schema.schema-version: '1.0'
         org.label-schema.vcs-ref: ${VCS_REF}
@@ -29,7 +29,7 @@ services:
         io.simcore.name: '{"name": "dy-static-file-server-dynamic-sidecar"}'
         io.simcore.outputs: '{"outputs": {"string_output": {"displayOrder": 1, "label": "String output", "description": "String value from input", "type": "string"}, "integer_output": {"displayOrder": 2, "label": "Integer output", "description": "Integer value from input", "type": "integer"}, "boolean_output": {"displayOrder": 3, "label": "Boolean output", "description": "Boolean value from input", "type": "boolean"}, "number_output": {"displayOrder": 4, "label": "Number output", "description": "Number value from input", "type": "number"}, "file_output": {"displayOrder": 5, "label": "File output", "description": "File from input", "type": "data:*/*", "fileToKeyMap": {"test_file": "file_output"}}}}'
         io.simcore.type: '{"type": "dynamic"}'
-        io.simcore.version: '{"version": "2.0.6"}'
+        io.simcore.version: '{"version": "2.0.7"}'
         org.label-schema.build-date: ${BUILD_DATE}
         org.label-schema.schema-version: '1.0'
         org.label-schema.vcs-ref: ${VCS_REF}
@@ -48,7 +48,7 @@ services:
         io.simcore.name: '{"name": "dy-static-file-server-dynamic-sidecar-compose-spec"}'
         io.simcore.outputs: '{"outputs": {"string_output": {"displayOrder": 1, "label": "String output", "description": "String value from input", "type": "string"}, "integer_output": {"displayOrder": 2, "label": "Integer output", "description": "Integer value from input", "type": "integer"}, "boolean_output": {"displayOrder": 3, "label": "Boolean output", "description": "Boolean value from input", "type": "boolean"}, "number_output": {"displayOrder": 4, "label": "Number output", "description": "Number value from input", "type": "number"}, "file_output": {"displayOrder": 5, "label": "File output", "description": "File from input", "type": "data:*/*", "fileToKeyMap": {"test_file": "file_output"}}}}'
         io.simcore.type: '{"type": "dynamic"}'
-        io.simcore.version: '{"version": "2.0.6"}'
+        io.simcore.version: '{"version": "2.0.7"}'
         org.label-schema.build-date: ${BUILD_DATE}
         org.label-schema.schema-version: '1.0'
         org.label-schema.vcs-ref: ${VCS_REF}

services/dy-static-file-server/metadata/metadata-dynamic-sidecar-compose-spec.yml

@@ -2,7 +2,7 @@ name: dy-static-file-server-dynamic-sidecar-compose-spec
 key: simcore/services/dynamic/dy-static-file-server-dynamic-sidecar-compose-spec
 type: dynamic
 integration-version: 1.0.0
-version: 2.0.6
+version: 2.0.7
 description: Modern test dynamic service providing a docker-compose specification file (with dynamic sidecar and compose-spec). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present.
 contact: [email protected]
 authors:

services/dy-static-file-server/metadata/metadata-dynamic-sidecar.yml

@@ -2,7 +2,7 @@ name: dy-static-file-server-dynamic-sidecar
 key: simcore/services/dynamic/dy-static-file-server-dynamic-sidecar
 type: dynamic
 integration-version: 1.0.0
-version: 2.0.6
+version: 2.0.7
 description: Modern test dynamic service (with dynamic sidecar). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present.
 contact: [email protected]
 authors:

services/dy-static-file-server/metadata/metadata.yml

@@ -2,7 +2,7 @@ name: dy-static-file-server
 key: simcore/services/dynamic/dy-static-file-server
 type: dynamic
 integration-version: 1.0.0
-version: 2.0.6
+version: 2.0.7
 description: Legacy test dynamic service (starts using original director-v0). The /workdir/generated-data directory is populated if no content is present.
 contact: [email protected]
 authors:

services/dy-static-file-server/src/dy_static_file_server/_oldinputs_to_outputs.py

@@ -0,0 +1,201 @@
+import logging
+import json
+import os
+import time
+from pathlib import Path
+from threading import Thread
+from typing import List, Optional
+from subprocess import check_output
+from watchdog.events import DirModifiedEvent, FileSystemEventHandler
+from watchdog.observers import Observer
+
+# when not testing `dy_static_file_server` directory is not detected
+# as a module; relative imports will not work
+try:
+    from index_html_generator import generate_index
+except ModuleNotFoundError:
+    from .index_html_generator import generate_index
+
+logger = logging.getLogger(__name__)
+
+
+class CouldNotDetectFilesException(Exception):
+    pass
+
+
+class UnifyingEventHandler(FileSystemEventHandler):
+    def __init__(self, input_dir: Path, output_dir: Path):
+        super().__init__()
+        self.input_dir: Path = input_dir
+        self.output_dir: Path = output_dir
+
+    def on_any_event(self, event: DirModifiedEvent):
+        super().on_any_event(event)
+        logger.info("Detected event: %s", event)
+        remap_input_to_output(self.input_dir, self.output_dir)
+        # alway regenerate index
+        generate_index()
+
+
+def _list_files_in_dir(path: Path) -> List[Path]:
+    return [x for x in path.rglob("*") if x.is_file()]
+
+
+def _log_files_in_path(path: Path) -> None:
+    split_command = f"ls -lah {path}".split(" ")
+    command_result = check_output(split_command).decode("utf-8")
+    logger.info("Files in '%s':\n%s", path, command_result)
+
+
+def _wait_for_paths_to_be_present_on_disk(
+    *paths: Path,
+    basedir: Path,
+    check_interval: float = 0.1,
+    max_attempts: int = 100,
+) -> None:
+    total_run_time = max_attempts * check_interval
+    logger.info(
+        "Will check for %s seconds every %s seconds", total_run_time, check_interval
+    )
+
+    for _ in range(max_attempts):
+        paths_are_not_missing = True
+        for path in paths:
+            if not path.exists():
+                paths_are_not_missing = False
+        logger.info("Are paths present? %s", paths_are_not_missing)
+
+        if paths_are_not_missing:
+            return
+        time.sleep(check_interval)
+
+    _log_files_in_path(basedir)
+    raise CouldNotDetectFilesException("Did not find expected files on disk!")
+
+
+def remap_input_to_output(input_dir: Path, output_dir: Path) -> None:
+    logger.info("Running directory sync")
+
+    input_file: Path = input_dir / "file_input" / "test_file"
+    inputs_key_values_file = input_dir / "key_values.json"
+
+    # When IO is slower the files may not already be present at the destination.
+    # Poll files to see when they are present or raise an error if missing
+    _wait_for_paths_to_be_present_on_disk(
+        input_file, inputs_key_values_file, basedir=input_dir
+    )
+
+    _log_files_in_path(input_dir)
+    _log_files_in_path(output_dir)
+
+    # remove all presnet files in outputs
+    files_in_output_dir = _list_files_in_dir(output_dir)
+    for output_file in files_in_output_dir:
+        output_file.unlink()
+
+    # move file to correct path
+    output_file_path: Path = output_dir / "file_output" / "test_file"
+    if input_file.is_file():
+        output_file_path.parent.mkdir(parents=True, exist_ok=True)
+        output_file_path.write_bytes(input_file.read_bytes())
+
+    # rewrite key_values.json
+    inputs_key_values = json.loads(inputs_key_values_file.read_text())
+    outputs_key_values = {
+        k.replace("_input", "_output"): v["value"] for k, v in inputs_key_values.items()
+    }
+    if input_file.is_file():
+        outputs_key_values["file_output"] = f"{output_file_path}"
+
+    outputs_key_values_file = output_dir / "key_values.json"
+    outputs_key_values_file.write_text(
+        json.dumps(outputs_key_values, sort_keys=True, indent=4)
+    )
+
+
+def get_path_from_env(env_var_name: str) -> Path:
+    str_path = os.environ.get(env_var_name, "")
+    if len(str_path) == 0:
+        raise ValueError(f"{env_var_name} could not be found or is empty")
+
+    path = Path(str_path)
+    if not path.is_dir():
+        raise ValueError(f"{env_var_name}={str_path} is not a valid dir path")
+    if not path.exists():
+        raise ValueError(f"{env_var_name}={str_path} does not exist")
+    return path
+
+
+class InputsObserver:
+    def __init__(self, input_dir: Path, output_dir: Path):
+        self.input_dir: Path = input_dir
+        self.output_dir: Path = output_dir
+
+        self._keep_running = True
+        self._thread: Optional[Thread] = None
+
+    def _runner(self) -> None:
+        event_handler = UnifyingEventHandler(
+            input_dir=self.input_dir, output_dir=self.output_dir
+        )
+        observer = Observer()
+        observer.schedule(event_handler, str(self.input_dir), recursive=True)
+        observer.start()
+        try:
+            logger.info("FolderMonitor started")
+            while self._keep_running:
+                time.sleep(0.5)
+        finally:
+            observer.stop()
+            observer.join()
+        logger.info("FolderMonitor stopped")
+
+    def start(self) -> None:
+        self._keep_running = True
+        self._thread = Thread(target=self._runner, daemon=True)
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._keep_running = False
+        if self._thread:
+            self._thread.join()
+            self._thread = None
+
+    def join(self) -> None:
+        if self._thread:
+            self._thread.join()
+        else:
+            raise RuntimeError(f"{self.__class__.__name__} was not started")
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+    is_legacy = os.environ.get("SIMCORE_NODE_BASEPATH", None) is not None
+
+    input_dir = get_path_from_env(
+        "INPUT_FOLDER" if is_legacy else "DY_SIDECAR_PATH_INPUTS"
+    )
+    output_dir = get_path_from_env(
+        "OUTPUT_FOLDER" if is_legacy else "DY_SIDECAR_PATH_OUTPUTS"
+    )
+    if input_dir == output_dir:
+        raise ValueError(f"Inputs and outputs directories match {input_dir}")
+
+    # make sure index exists before the monitor starts
+    generate_index()
+
+    # start monitor that copies files in case they something
+    inputs_objserver = InputsObserver(input_dir, output_dir)
+    inputs_objserver.start()
+    inputs_objserver.join()
+
+    logger.info("%s main exited", InputsObserver.__name__)
+
+
+if __name__ == "__main__":
+    main()

services/dy-static-file-server/src/dy_static_file_server/inputs_to_outputs.py

@@ -97,8 +97,8 @@ def _get_observed_state(self) -> Dict[Path, Dict[Path, Tuple[str, float]]]:
 
     async def _monitor(self) -> None:
 
-        _logger.info("Started monitor")
-        previous_state = self._get_observed_state()
+        _logger.info("Started monitor on %s", self.to_observe)
+        previous_state = {}  # always trigger once  a sync
 
         while self._keep_running:
             await asyncio.sleep(self.monitor_interval)

services/dy-static-file-server/versioning/service.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.0.6
+current_version = 2.0.7
 commit = False
 message = service/kernel version: {current_version} → {new_version}
 tag = False

toc.json

@@ -76,7 +76,7 @@
     "name": "dy-static-file-server",
     "simcore.service.settings": "[{\"name\": \"resources\", \"type\": \"Resources\", \"value\": {\"mem_limit\":268435456, \"cpu_limit\": 10000000}}, {\"name\": \"ports\", \"type\": \"int\", \"value\": 8080}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}]",
     "type": "dynamic",
-    "version": "2.0.6"
+    "version": "2.0.7"
   },
   "dy-static-file-server-dynamic-sidecar": {
     "description": "Modern test dynamic service (with dynamic sidecar). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present.",
@@ -86,7 +86,7 @@
     "name": "dy-static-file-server-dynamic-sidecar",
     "simcore.service.settings": "[{\"name\": \"resources\", \"type\": \"Resources\", \"value\": {\"mem_limit\":268435456, \"cpu_limit\": 10000000}}, {\"name\": \"ports\", \"type\": \"int\", \"value\": 8080}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}]",
     "type": "dynamic",
-    "version": "2.0.6"
+    "version": "2.0.7"
   },
   "dy-static-file-server-dynamic-sidecar-compose-spec": {
     "description": "Modern test dynamic service providing a docker-compose specification file (with dynamic sidecar and compose-spec). Changes to the inputs will be forwarded to the outputs. The /workdir/generated-data directory is populated if no content is present.",
@@ -96,7 +96,7 @@
     "name": "dy-static-file-server-dynamic-sidecar-compose-spec",
     "simcore.service.settings": "[{\"name\": \"resources\", \"type\": \"Resources\", \"value\": {\"mem_limit\":268435456, \"cpu_limit\": 10000000}}, {\"name\": \"ports\", \"type\": \"int\", \"value\": 8080}, {\"name\": \"constraints\", \"type\": \"string\", \"value\": [\"node.platform.os == linux\"]}]",
     "type": "dynamic",
-    "version": "2.0.6"
+    "version": "2.0.7"
   },
   "jupyter-base-notebook": {
     "description": "Jupyter notebook",