feat(arcor2_build): cross imports

src/docker/arcor2_build/BUILD

@@ -1 +1 @@
-docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.2.0"])
+docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.3.0"])

src/python/arcor2/CHANGELOG.md

@@ -2,6 +2,12 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+## [1.2.0] - WIP
+
+### Added
+
+- `save_type_def` helper function.
+
 ## [1.1.0] - 2023-07-20
 
 ### Added

src/python/arcor2/VERSION

@@ -1 +1 @@
-1.1.0
+1.2.0

src/python/arcor2/helpers.py

@@ -93,14 +93,14 @@ async def run_in_executor(
 T = TypeVar("T")
 
 
-def save_and_import_type_def(source: str, type_name: str, output_type: type[T], path: str, module_name: str) -> type[T]:
+def save_type_def(source: str, type_name: str, path: str, module_name: str) -> str:
     """Save source to a file in object_type_path directory.
 
     :param source:
     :param type_name:
-    :param output_type:
-    :param object_type_path:
-    :return:
+    :param path:
+    :param module_name:
+    :return: full path to the module
     """
 
     type_file = humps.depascalize(type_name)
@@ -109,6 +109,22 @@ def save_and_import_type_def(source: str, type_name: str, output_type: type[T],
     with open(full_path, "w") as file:
         file.write(source)
 
+    return full_path
+
+
+def save_and_import_type_def(source: str, type_name: str, output_type: type[T], path: str, module_name: str) -> type[T]:
+    """Save source to a file in object_type_path directory and tries to import
+    it.
+
+    :param source:
+    :param type_name:
+    :param output_type:
+    :param object_type_path:
+    :return:
+    """
+
+    full_path = save_type_def(source, type_name, path, module_name)
+
     try:
         return import_type_def(type_name, output_type, path, module_name)
     except Arcor2Exception:

src/python/arcor2_build/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+## [1.3.0] - WIP
+
+### Changed
+
+- Object Types listed in `project_objects_ids` are now automatically added to the Execution package.
+  - Those objects don't have to be based on `Generic`, actually the file may contain whatever (no need for a class with name corresponding to the name of the module / Object Type).
+- All scene objects are downloaded prior to trying to find their predecessors. This will allow importing code from one Object Type into another.
+- Cross-importing should be done only between OT listed in scene or objects listed in `project_objects_ids` - OT should not import something from another OT predecessor, as this situation can't be figured out. 
+- All changes are fully backwards compatible.
+
 ## [1.2.0] - 2024-01-08
 
 ### Changed

src/python/arcor2_build/VERSION

@@ -1 +1 @@
-1.2.0
+1.3.0

src/python/arcor2_build/scripts/build.py

@@ -28,7 +28,7 @@
 from arcor2.data.object_type import Models, ObjectModel, ObjectType
 from arcor2.exceptions import Arcor2Exception
 from arcor2.flask import RespT, create_app, run_app
-from arcor2.helpers import port_from_url, save_and_import_type_def
+from arcor2.helpers import port_from_url, save_and_import_type_def, save_type_def
 from arcor2.logging import get_logger
 from arcor2.object_types.abstract import Generic
 from arcor2.object_types.utils import base_from_source, built_in_types_names, prepare_object_types_dir
@@ -154,6 +154,18 @@ def _publish(project_id: str, package_name: str) -> RespT:
                 obj_types = set(cached_scene.object_types)
                 obj_types_with_models: set[str] = set()
 
+                # this is kinda workaround allowing adding additional/utility Python modules into Execution package
+                if project.project_objects_ids:
+                    for additional_module_id in project.project_objects_ids:
+                        logger.debug(f"Getting additional module {additional_module_id}.")
+                        am = ps.get_object_type(additional_module_id)
+                        save_type_def(am.source, am.id, tmp_dir, OBJECT_TYPE_MODULE)
+
+                # to allow imports between OTs, all objects listed in scene are downloaded first
+                for scene_obj in scene.objects:
+                    obj_type = ps.get_object_type(scene_obj.type)
+                    save_type_def(obj_type.source, obj_type.id, tmp_dir, OBJECT_TYPE_MODULE)
+
                 for scene_obj in scene.objects:
                     if scene_obj.type in types_dict:
                         continue

src/python/arcor2_build/tests/BUILD

@@ -1,5 +1,6 @@
 python_tests(
     runtime_package_dependencies=[
         "src/python/arcor2_build/scripts:build",
+        "src/python/arcor2_mocks/scripts:mock_project",
     ]
 )

src/python/arcor2_build/tests/test_cross_import.py

@@ -0,0 +1,118 @@
+import os
+import subprocess as sp
+import tempfile
+import time
+from typing import Iterator
+
+import pytest
+
+from arcor2 import rest
+from arcor2.clients import project_service as ps
+from arcor2.data.common import Project, Scene, SceneObject
+from arcor2.data.object_type import ObjectType
+from arcor2.helpers import find_free_port
+
+build_url = ""
+
+
+def finish_processes(processes) -> None:
+    for proc in processes:
+        proc.terminate()
+        proc.wait()
+        print(proc.communicate()[0].decode("utf-8").strip())
+
+
+def check_health(service_name: str, service_url: str, timeout: int = 60) -> None:
+    for _ in range(timeout):
+        try:
+            rest.call(rest.Method.GET, f"{service_url}/healthz/ready")  # to check whether the service is running
+            break
+        except rest.RestException:
+            pass
+        time.sleep(1)
+    else:
+        raise Exception(f"{service_name} service at {service_url} is not responding.")
+
+
+@pytest.fixture()
+def start_processes() -> Iterator[None]:
+    global build_url
+
+    my_env = os.environ.copy()
+    kwargs = {"env": my_env, "stdout": sp.PIPE, "stderr": sp.STDOUT}
+
+    project_port = find_free_port()
+    project_url = f"http://0.0.0.0:{project_port}"
+    my_env["ARCOR2_PROJECT_SERVICE_URL"] = project_url
+    my_env["ARCOR2_PROJECT_SERVICE_MOCK_PORT"] = str(project_port)
+    ps.URL = project_url
+
+    processes = []
+
+    processes.append(sp.Popen(["python", "src.python.arcor2_mocks.scripts/mock_project.pex"], **kwargs))  # type: ignore
+
+    check_health("Project", project_url)
+
+    build_url = f"http://0.0.0.0:{find_free_port()}"
+    my_env["ARCOR2_BUILD_URL"] = build_url
+    my_env["ARCOR2_PROJECT_PATH"] = "whatever"
+    processes.append(
+        sp.Popen(["python", "src.python.arcor2_build.scripts/build.pex", "--debug"], **kwargs)  # type: ignore
+    )
+    check_health("Build", build_url)
+
+    yield None
+
+    finish_processes(processes)
+
+
+additional_module = """
+def whatever():
+    return "whatever"
+"""
+
+ot1 = """
+from arcor2.object_types.abstract import Generic
+from .additional_module import whatever
+
+class ObjectTypeOne(Generic):
+    pass
+"""
+
+ot2 = """
+from arcor2.object_types.abstract import Generic
+from .object_type_one import ObjectTypeOne
+from .additional_module import whatever
+
+class ObjectTypeTwo(Generic):
+    pass
+"""
+
+
+def test_cross_import(start_processes: None) -> None:
+    ps.update_object_type(ObjectType("AdditionalModule", additional_module))
+    ps.update_object_type(ObjectType("ObjectTypeOne", ot1))
+    ps.update_object_type(ObjectType("ObjectTypeTwo", ot2))
+
+    scene = Scene("test_scene")
+    scene.objects.append(SceneObject("ot1", "ObjectTypeOne"))
+    scene.objects.append(SceneObject("ot2", "ObjectTypeTwo"))
+
+    ps.update_scene(scene)
+
+    project = Project("test_project", scene.id)
+    project.project_objects_ids = ["AdditionalModule"]
+    project.has_logic = False
+    ps.update_project(project)
+
+    with tempfile.TemporaryDirectory() as tmpdirname:
+        path = os.path.join(tmpdirname, "publish.zip")
+
+        rest.download(
+            f"{build_url}/project/publish",
+            path,
+            {
+                "packageName": "test_package",
+                "projectId": project.id,
+            },
+        )