fix(arcor2_build): extra objects were not added into package

README.md

@@ -54,7 +54,7 @@ The following video by [Kinali](https://www.kinali.cz/en/) shows the use case (o
 
 [README](src/python/arcor2_build/README.md) | [CHANGELOG](src/python/arcor2_build/CHANGELOG.md)
 
- - 2024-01-26: [1.3.0](https://github.com/robofit/arcor2/releases/tag/arcor2_build%2F1.3.0) ([docker](https://hub.docker.com/r/arcor2/arcor2_build/tags?page=1&ordering=last_updated&name=1.3.0), [pypi](https://pypi.org/project/arcor2-build/1.3.0/)).
+ - 2024-01-30: [1.3.1](https://github.com/robofit/arcor2/releases/tag/arcor2_build%2F1.3.1) ([docker](https://hub.docker.com/r/arcor2/arcor2_build/tags?page=1&ordering=last_updated&name=1.3.1), [pypi](https://pypi.org/project/arcor2-build/1.3.1/)).
 
 ### arcor2_build_data
 

compose-files/fit-demo/docker-compose.yml

@@ -44,7 +44,7 @@ services:
       - ARCOR2_KINECT_AZURE_URL=http://192.168.104.100:5017 # Run kinect using pants
 
   fit-demo-build:
-    image: arcor2/arcor2_build:1.2.0
+    image: arcor2/arcor2_build:1.3.1
     container_name: fit-demo-build
     depends_on:
       fit-demo-project:

src/docker/arcor2_build/BUILD

@@ -1 +1 @@
-docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.3.0"])
+docker_image(name="arcor2_build", repository="arcor2/arcor2_build", image_tags=["1.3.1"])

src/python/arcor2_build/CHANGELOG.md

@@ -2,6 +2,16 @@
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+## [1.3.1] - 2024-01-30
+
+### Fixed
+
+- Objects specified in `project_objects_ids` were actually missing in the package.
+
+### Changes
+
+- Support for `project_objects_ids` was added for `import`.
+
 ## [1.3.0] - 2024-01-26
 
 ### Changed

src/python/arcor2_build/VERSION

@@ -1 +1 @@
-1.3.0
+1.3.1

src/python/arcor2_build/scripts/build.py

@@ -160,6 +160,7 @@ def _publish(project_id: str, package_name: str) -> RespT:
                         logger.debug(f"Getting additional module {additional_module_id}.")
                         am = ps.get_object_type(additional_module_id)
                         save_type_def(am.source, am.id, tmp_dir, OBJECT_TYPE_MODULE)
+                        zf.writestr(os.path.join(ot_path, humps.depascalize(additional_module_id)) + ".py", am.source)
 
                 # to allow imports between OTs, all objects listed in scene are downloaded first
                 for scene_obj in scene.objects:
@@ -412,6 +413,24 @@ def project_import() -> RespT:
 
             prepare_object_types_dir(tmp_dir, OBJECT_TYPE_MODULE)
 
+            # read and save additional objects...
+            if project.project_objects_ids:
+                for add_obj in project.project_objects_ids:
+                    try:
+                        src = read_str_from_zip(zip_file, f"object_types/{humps.depascalize(add_obj)}.py")
+                    except KeyError:
+                        raise NotFound(f"Additional object {add_obj} is missing in the package.")
+                    objects[add_obj] = ObjectType(add_obj, src)
+                    save_type_def(src, add_obj, tmp_dir, OBJECT_TYPE_MODULE)
+
+            # extra pass through scene objects to save them all - in order to allow imports between OTs
+            for scene_obj in scene.objects:
+                try:
+                    obj_type_src = read_str_from_zip(zip_file, f"object_types/{humps.depascalize(scene_obj.type)}.py")
+                except KeyError:
+                    raise NotFound(f"Object type {scene_obj.type} is missing in the package.")
+                save_type_def(obj_type_src, scene_obj.type, tmp_dir, OBJECT_TYPE_MODULE)
+
             for scene_obj in scene.objects:
                 obj_type_name = scene_obj.type
 
@@ -462,7 +481,7 @@ def project_import() -> RespT:
 
             models[obj_type.id] = model
 
-        if not project.has_logic:
+        if not project.has_logic or update_project_from_script:
             logger.debug("Importing the main script.")
 
             try:
@@ -475,13 +494,6 @@ def project_import() -> RespT:
             except Arcor2Exception:
                 raise InvalidPackage("Invalid code of the main script.")
 
-        # case when preparing data from script to decompilation
-        if update_project_from_script:
-            try:
-                src = zip_file.read("script.py").decode("UTF-8")
-            except KeyError:
-                raise NotFound("Could not find script.py.")
-
     # check that we are not going to overwrite something
     if not overwrite_scene:
         try:
@@ -537,8 +549,8 @@ def project_import() -> RespT:
 
     if update_project_from_script:
         logger.debug("Decompiling source...")
-        project = python_to_json(project, scene, src, object_type)
-        logger.debug("Decompile was successfull and project was overwritten")
+        project = python_to_json(project, scene, script, object_type)
+        logger.debug("Decompilation was successfull and project was overwritten")
 
     for model in models.values():
         ps.put_model(model)

src/python/arcor2_build/tests/test_cross_import.py

@@ -2,13 +2,14 @@
 import subprocess as sp
 import tempfile
 import time
+import zipfile
 from typing import Iterator
 
 import pytest
 
 from arcor2 import rest
 from arcor2.clients import project_service as ps
-from arcor2.data.common import Project, Scene, SceneObject
+from arcor2.data.common import Project, ProjectSources, Scene, SceneObject
 from arcor2.data.object_type import ObjectType
 from arcor2.helpers import find_free_port
 
@@ -76,7 +77,7 @@ def whatever():
 from .additional_module import whatever
 
 class ObjectTypeOne(Generic):
-    pass
+    _ABSTRACT = False
 """
 
 ot2 = """
@@ -85,7 +86,7 @@ class ObjectTypeOne(Generic):
 from .additional_module import whatever
 
 class ObjectTypeTwo(Generic):
-    pass
+    _ABSTRACT = False
 """
 
 
@@ -104,6 +105,7 @@ def test_cross_import(start_processes: None) -> None:
     project.project_objects_ids = ["AdditionalModule"]
     project.has_logic = False
     ps.update_project(project)
+    ps.update_project_sources(ProjectSources(project.id, "blah"))
 
     with tempfile.TemporaryDirectory() as tmpdirname:
         path = os.path.join(tmpdirname, "publish.zip")
@@ -116,3 +118,19 @@ def test_cross_import(start_processes: None) -> None:
                 "projectId": project.id,
             },
         )
+
+        with zipfile.ZipFile(path) as zip_file:
+            ot_dir_list = [name for name in zip_file.namelist() if name.startswith("object_types")]
+            # there should be three OTs and __init__.py
+            assert len(ot_dir_list) == 4, f"Strange content of object_types dir: {ot_dir_list}"
+
+        assert {ot.id for ot in ps.get_object_type_ids()} == {"AdditionalModule", "ObjectTypeOne", "ObjectTypeTwo"}
+
+        ps.delete_object_type("AdditionalModule")
+        ps.delete_object_type("ObjectTypeOne")
+        ps.delete_object_type("ObjectTypeTwo")
+
+        with open(path, "rb") as fh:
+            rest.call(rest.Method.PUT, url=f"{build_url}/project/import", files={"executionPackage": fh.read()})
+
+        assert {ot.id for ot in ps.get_object_type_ids()} == {"AdditionalModule", "ObjectTypeOne", "ObjectTypeTwo"}

src/python/arcor2_mocks/scripts/mock_project.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
 
 import argparse
-import copy
-import random
 from datetime import datetime, timezone
 
 import humps
@@ -18,6 +16,7 @@
 
 SCENES: dict[str, common.Scene] = {}
 PROJECTS: dict[str, common.Project] = {}
+PROJECT_SOURCES: dict[str, common.ProjectSources] = {}
 OBJECT_TYPES: dict[str, object_type.ObjectType] = {}
 
 BOXES: dict[str, object_type.Box] = {}
@@ -76,6 +75,80 @@ def put_project() -> RespT:
     return jsonify(project.modified.isoformat())
 
 
+@app.route("/projects/sources", methods=["PUT"])
+def put_project_sources() -> RespT:
+    """Add or update project sources.
+    ---
+    put:
+        tags:
+            - Project
+        description: Add or update project sources.
+        requestBody:
+              content:
+                application/json:
+                  schema:
+                    $ref: ProjectSources
+        responses:
+            200:
+              description: Timestamp of last project modification.
+            500:
+              description: "Error types: **General**, **ProjectGeneral**, **NotFound**."
+              content:
+                application/json:
+                  schema:
+                    $ref: WebApiError
+    """
+
+    if not isinstance(request.json, dict):
+        raise ProjectGeneral("Body should be a JSON dict containing ProjectSources.")
+
+    project_sources = common.ProjectSources.from_dict(humps.decamelize(request.json))
+
+    if project_sources.id not in PROJECTS:
+        raise NotFound(f"Project {project_sources.id} does not exist.")
+
+    PROJECT_SOURCES[project_sources.id] = project_sources
+    return Response(status=200)
+
+
+@app.route("/projects/<string:id>/sources", methods=["GET"])
+def get_project_sources(id: str) -> RespT:
+    """Get project sources.
+    ---
+    get:
+        tags:
+            - Project
+        summary: Gets project by project id.
+        parameters:
+            - name: id
+              in: path
+              description: unique ID
+              required: true
+              schema:
+                type: string
+        responses:
+            200:
+              description: Ok
+              content:
+                application/json:
+                  schema:
+                    $ref: ProjectSources
+            500:
+              description: "Error types: **General**, **NotFound**."
+              content:
+                application/json:
+                  schema:
+                    $ref: WebApiError
+    """
+
+    try:
+        project_sources = PROJECT_SOURCES[id]
+    except KeyError:
+        raise NotFound(f"Sources for project {id} not found.")
+
+    return jsonify(humps.camelize(project_sources.to_dict()))
+
+
 @app.route("/projects/<string:id>", methods=["GET"])
 def get_project(id: str) -> RespT:
     """Add or update project.
@@ -107,16 +180,11 @@ def get_project(id: str) -> RespT:
     """
 
     try:
-        project_copy = copy.deepcopy(PROJECTS[id])
+        project = PROJECTS[id]
     except KeyError:
         raise NotFound(f"Project {id} was not found.")
 
-    random.shuffle(project_copy.action_points)
-    random.shuffle(project_copy.logic)
-    random.shuffle(project_copy.object_overrides)
-    random.shuffle(project_copy.parameters)
-
-    return jsonify(humps.camelize(project_copy.to_dict()))
+    return jsonify(humps.camelize(project.to_dict()))
 
 
 @app.route("/projects/<string:id>", methods=["DELETE"])
@@ -274,13 +342,11 @@ def get_scene(id: str) -> RespT:
     """
 
     try:
-        scene_copy = copy.deepcopy(SCENES[id])
+        scene = SCENES[id]
     except KeyError:
         raise NotFound(f"Scene {id} was not found.")
 
-    random.shuffle(scene_copy.objects)
-
-    return jsonify(humps.camelize(scene_copy.to_dict()))
+    return jsonify(humps.camelize(scene.to_dict()))
 
 
 @app.route("/scenes/<string:id>", methods=["DELETE"])
@@ -881,6 +947,7 @@ def main() -> None:
         PROJECT_PORT,
         [
             common.Project,
+            common.ProjectSources,
             common.Scene,
             common.IdDesc,
             object_type.ObjectType,