diff --git a/src/docker/arcor2_build/BUILD b/src/docker/arcor2_build/BUILD index a9444ac33..117153015 100644 --- a/src/docker/arcor2_build/BUILD +++ b/src/docker/arcor2_build/BUILD @@ -1 +1 @@ -docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.2.0"]) +docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.3.0"]) diff --git a/src/python/arcor2/CHANGELOG.md b/src/python/arcor2/CHANGELOG.md index 1695ca0c4..c6d2a6162 100644 --- a/src/python/arcor2/CHANGELOG.md +++ b/src/python/arcor2/CHANGELOG.md @@ -2,6 +2,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +## [1.2.0] - WIP + +### Added + +- `save_type_def` helper function. + ## [1.1.0] - 2023-07-20 ### Added diff --git a/src/python/arcor2/VERSION b/src/python/arcor2/VERSION index 1cc5f657e..867e52437 100644 --- a/src/python/arcor2/VERSION +++ b/src/python/arcor2/VERSION @@ -1 +1 @@ -1.1.0 \ No newline at end of file +1.2.0 \ No newline at end of file diff --git a/src/python/arcor2/helpers.py b/src/python/arcor2/helpers.py index a3ab801b9..459228358 100644 --- a/src/python/arcor2/helpers.py +++ b/src/python/arcor2/helpers.py @@ -93,14 +93,14 @@ async def run_in_executor( T = TypeVar("T") -def save_and_import_type_def(source: str, type_name: str, output_type: type[T], path: str, module_name: str) -> type[T]: +def save_type_def(source: str, type_name: str, path: str, module_name: str) -> str: """Save source to a file in object_type_path directory. :param source: :param type_name: - :param output_type: - :param object_type_path: - :return: + :param path: + :param module_name: + :return: full path to the module """ type_file = humps.depascalize(type_name) @@ -109,6 +109,22 @@ def save_and_import_type_def(source: str, type_name: str, output_type: type[T], with open(full_path, "w") as file: file.write(source) + return full_path + + +def save_and_import_type_def(source: str, type_name: str, output_type: type[T], path: str, module_name: str) -> type[T]: + """Save source to a file in object_type_path directory and tries to import + it. + + :param source: + :param type_name: + :param output_type: + :param object_type_path: + :return: + """ + + full_path = save_type_def(source, type_name, path, module_name) + try: return import_type_def(type_name, output_type, path, module_name) except Arcor2Exception: diff --git a/src/python/arcor2_build/CHANGELOG.md b/src/python/arcor2_build/CHANGELOG.md index 152ce4b3c..bc8cebb36 100644 --- a/src/python/arcor2_build/CHANGELOG.md +++ b/src/python/arcor2_build/CHANGELOG.md @@ -2,6 +2,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +## [1.3.0] - WIP + +### Changed + +- Object Types listed in `project_objects_ids` are now automatically added to the Execution package. + - Those objects don't have to be based on `Generic`, actually the file may contain whatever (no need for a class with name corresponding to the name of the module / Object Type). +- All scene objects are downloaded prior to trying to find their predecessors. This will allow importing code from one Object Type into another. +- Cross-importing should be done only between OT listed in scene or objects listed in `project_objects_ids` - OT should not import something from another OT predecessor, as this situation can't be figured out. +- All changes are fully backwards compatible. + ## [1.2.0] - 2024-01-08 ### Changed diff --git a/src/python/arcor2_build/VERSION b/src/python/arcor2_build/VERSION index 867e52437..589268e6f 100644 --- a/src/python/arcor2_build/VERSION +++ b/src/python/arcor2_build/VERSION @@ -1 +1 @@ -1.2.0 \ No newline at end of file +1.3.0 \ No newline at end of file diff --git a/src/python/arcor2_build/scripts/build.py b/src/python/arcor2_build/scripts/build.py index e6b18baa4..d9eccfd60 100644 --- a/src/python/arcor2_build/scripts/build.py +++ b/src/python/arcor2_build/scripts/build.py @@ -28,7 +28,7 @@ from arcor2.data.object_type import Models, ObjectModel, ObjectType from arcor2.exceptions import Arcor2Exception from arcor2.flask import RespT, create_app, run_app -from arcor2.helpers import port_from_url, save_and_import_type_def +from arcor2.helpers import port_from_url, save_and_import_type_def, save_type_def from arcor2.logging import get_logger from arcor2.object_types.abstract import Generic from arcor2.object_types.utils import base_from_source, built_in_types_names, prepare_object_types_dir @@ -154,6 +154,18 @@ def _publish(project_id: str, package_name: str) -> RespT: obj_types = set(cached_scene.object_types) obj_types_with_models: set[str] = set() + # this is kinda workaround allowing adding additional/utility Python modules into Execution package + if project.project_objects_ids: + for additional_module_id in project.project_objects_ids: + logger.debug(f"Getting additional module {additional_module_id}.") + am = ps.get_object_type(additional_module_id) + save_type_def(am.source, am.id, tmp_dir, OBJECT_TYPE_MODULE) + + # to allow imports between OTs, all objects listed in scene are downloaded first + for scene_obj in scene.objects: + obj_type = ps.get_object_type(scene_obj.type) + save_type_def(obj_type.source, obj_type.id, tmp_dir, OBJECT_TYPE_MODULE) + for scene_obj in scene.objects: if scene_obj.type in types_dict: continue diff --git a/src/python/arcor2_build/tests/BUILD b/src/python/arcor2_build/tests/BUILD index 7f986266b..c527924c5 100644 --- a/src/python/arcor2_build/tests/BUILD +++ b/src/python/arcor2_build/tests/BUILD @@ -1,5 +1,6 @@ python_tests( runtime_package_dependencies=[ "src/python/arcor2_build/scripts:build", + "src/python/arcor2_mocks/scripts:mock_project", ] ) diff --git a/src/python/arcor2_build/tests/test_cross_import.py b/src/python/arcor2_build/tests/test_cross_import.py new file mode 100644 index 000000000..f16af693b --- /dev/null +++ b/src/python/arcor2_build/tests/test_cross_import.py @@ -0,0 +1,118 @@ +import os +import subprocess as sp +import tempfile +import time +from typing import Iterator + +import pytest + +from arcor2 import rest +from arcor2.clients import project_service as ps +from arcor2.data.common import Project, Scene, SceneObject +from arcor2.data.object_type import ObjectType +from arcor2.helpers import find_free_port + +build_url = "" + + +def finish_processes(processes) -> None: + for proc in processes: + proc.terminate() + proc.wait() + print(proc.communicate()[0].decode("utf-8").strip()) + + +def check_health(service_name: str, service_url: str, timeout: int = 60) -> None: + for _ in range(timeout): + try: + rest.call(rest.Method.GET, f"{service_url}/healthz/ready") # to check whether the service is running + break + except rest.RestException: + pass + time.sleep(1) + else: + raise Exception(f"{service_name} service at {service_url} is not responding.") + + +@pytest.fixture() +def start_processes() -> Iterator[None]: + global build_url + + my_env = os.environ.copy() + kwargs = {"env": my_env, "stdout": sp.PIPE, "stderr": sp.STDOUT} + + project_port = find_free_port() + project_url = f"http://0.0.0.0:{project_port}" + my_env["ARCOR2_PROJECT_SERVICE_URL"] = project_url + my_env["ARCOR2_PROJECT_SERVICE_MOCK_PORT"] = str(project_port) + ps.URL = project_url + + processes = [] + + processes.append(sp.Popen(["python", "src.python.arcor2_mocks.scripts/mock_project.pex"], **kwargs)) # type: ignore + + check_health("Project", project_url) + + build_url = f"http://0.0.0.0:{find_free_port()}" + my_env["ARCOR2_BUILD_URL"] = build_url + my_env["ARCOR2_PROJECT_PATH"] = "whatever" + processes.append( + sp.Popen(["python", "src.python.arcor2_build.scripts/build.pex", "--debug"], **kwargs) # type: ignore + ) + check_health("Build", build_url) + + yield None + + finish_processes(processes) + + +additional_module = """ +def whatever(): + return "whatever" +""" + +ot1 = """ +from arcor2.object_types.abstract import Generic +from .additional_module import whatever + +class ObjectTypeOne(Generic): + pass +""" + +ot2 = """ +from arcor2.object_types.abstract import Generic +from .object_type_one import ObjectTypeOne +from .additional_module import whatever + +class ObjectTypeTwo(Generic): + pass +""" + + +def test_cross_import(start_processes: None) -> None: + ps.update_object_type(ObjectType("AdditionalModule", additional_module)) + ps.update_object_type(ObjectType("ObjectTypeOne", ot1)) + ps.update_object_type(ObjectType("ObjectTypeTwo", ot2)) + + scene = Scene("test_scene") + scene.objects.append(SceneObject("ot1", "ObjectTypeOne")) + scene.objects.append(SceneObject("ot2", "ObjectTypeTwo")) + + ps.update_scene(scene) + + project = Project("test_project", scene.id) + project.project_objects_ids = ["AdditionalModule"] + project.has_logic = False + ps.update_project(project) + + with tempfile.TemporaryDirectory() as tmpdirname: + path = os.path.join(tmpdirname, "publish.zip") + + rest.download( + f"{build_url}/project/publish", + path, + { + "packageName": "test_package", + "projectId": project.id, + }, + )