Merge branch 'master' of https://github.com/zdenekm/arcor2

.circleci/config.yml

@@ -17,8 +17,12 @@ jobs:
 
     working_directory: ~/repo
 
+    environment:
+      ARCOR2_PROJECT_PATH: /tmp/arcor2_project
+
     steps:
       - checkout
+      - setup_remote_docker
 
       # Download and cache dependencies
       - restore_cache:
@@ -27,12 +31,20 @@ jobs:
             # fallback to using the latest cache if no exact match is found
             - v1-dependencies-
 
+      - run:
+          name: prepare environment
+          command: |
+            mkdir -p ${ARCOR2_PROJECT_PATH}
+
       - run:
           name: install dependencies
           command: |
             python3 -m venv venv
             . venv/bin/activate
-            pip install -r requirements.txt
+            sudo apt-get update
+            sudo apt-get install build-essential
+            pip install --upgrade pip setuptools
+            pip3 install -e .[test]
             pip install pytest flake8 mypy
 
       - save_cache:
@@ -56,7 +68,7 @@ jobs:
           name: run tests
           command: |
             . venv/bin/activate
-            py.test arcor2
+            py.test -m "not integration" --docker-compose arcor2/docker arcor2
 
       - store_artifacts:
           path: test-reports

arcor2/VERSION

@@ -0,0 +1 @@
+0.1.0

arcor2/__init__.py

@@ -1,5 +1,6 @@
 from typing import Dict, Set, Tuple
 import warnings
+import pkgutil
 
 # key: name of parameter, value: name of method to call (to get set of strings), set of parent parameters
 DynamicParamDict = Dict[str, Tuple[str, Set[str]]]
@@ -16,3 +17,10 @@
     module='dataclasses_jsonschema',
     message="Unable to create schema for 'Any'"
 )
+
+
+def version() -> str:
+    res = pkgutil.get_data(__name__, 'VERSION')
+    if not res:
+        return "unknown"
+    return res.decode().strip()

arcor2/aio_persistent_storage.py

@@ -3,7 +3,7 @@
 from arcor2.data.services import ServiceType
 from arcor2.helpers import run_in_executor
 from arcor2 import persistent_storage
-from arcor2.persistent_storage import PersistentStorageException
+from arcor2.persistent_storage import PersistentStorageException  # noqa
 
 
 async def get_mesh(mesh_id: str) -> Mesh:

arcor2/data/tests/test_utils.py

@@ -1,4 +1,4 @@
-from openapi_spec_validator import validate_spec
+from openapi_spec_validator import validate_spec  # type: ignore
 import yaml
 from arcor2.data import utils
 

arcor2/nodes/builder.py

@@ -5,40 +5,37 @@
 import tempfile
 import shutil
 import argparse
+import json
 
 from typing import Set
 
 from apispec import APISpec  # type: ignore
 from apispec_webframeworks.flask import FlaskPlugin  # type: ignore
 from flask import Flask, send_file
-from dataclasses_jsonschema.apispec import DataclassesPlugin
+from flask_swagger_ui import get_swaggerui_blueprint  # type: ignore
 
+import arcor2
 from arcor2 import persistent_storage as ps
-from arcor2.source.logic import program_src, get_logic_from_source
+from arcor2.source.logic import program_src  # , get_logic_from_source
 from arcor2.source.utils import derived_resources_class, global_actions_class
 from arcor2.source import SourceException
 from arcor2.object_types_utils import built_in_types_names
 from arcor2.helpers import camel_case_to_snake_case
 from arcor2.data.object_type import ObjectModel
 
 PORT = 5007
+SERVICE_NAME = "ARCOR2 Build Service"
 
 # Create an APISpec
 spec = APISpec(
-    title="ARCOR2 Builder Service",
-    version="0.0.1",
+    title=SERVICE_NAME,
+    version=arcor2.version(),
     openapi_version="3.0.2",
-    plugins=[FlaskPlugin(), DataclassesPlugin()],
+    plugins=[FlaskPlugin()],
 )
 
-# Dependant schemas are added automatically
-# spec.components.schema(IdDescList.__name__, schema=IdDescList)
-
 app = Flask(__name__)
 
-app.config['APISPEC_SPEC'] = spec
-app.config['APISPEC_SWAGGER_URL'] = '/swagger/'
-
 
 @app.route("/project/<string:project_id>/publish", methods=['GET'])
 def project_publish(project_id: str):
@@ -156,14 +153,20 @@ def project_script(project_id: str):
     """Project script
             ---
             put:
-              description: Add or update project main script
-              consumes:
-                 - multipart/form-data
+              description: Add or update project main script (DOES NOT WORK YET).
               parameters:
-                 - in: formData
-                   name: upfile
-                   type: file
-                   description: The file to upload.
+                - in: path
+                  name: project_id
+                  schema:
+                    type: string
+                  required: true
+                  description: unique ID
+              requestBody:
+                  content:
+                    text/x-python:
+                      schema:
+                        type: string
+                        format: binary
               responses:
                 200:
                   description: Ok
@@ -172,20 +175,36 @@ def project_script(project_id: str):
     pass
 
 
+@app.route("/swagger/api/swagger.json", methods=["GET"])
+def get_swagger():
+    return json.dumps(spec.to_dict())
+
+
 with app.test_request_context():
     spec.path(view=project_publish)
+    spec.path(view=project_script)
 
 
 def main():
 
-    parser = argparse.ArgumentParser(description='ARCOR2 Project Builder')
+    parser = argparse.ArgumentParser(description=SERVICE_NAME)
     parser.add_argument('-s', '--swagger', action="store_true", default=False)
     args = parser.parse_args()
 
     if args.swagger:
         print(spec.to_yaml())
         return
 
+    SWAGGER_URL = "/swagger"
+
+    swaggerui_blueprint = get_swaggerui_blueprint(
+        SWAGGER_URL,  # Swagger UI static files will be mapped to '{SWAGGER_URL}/dist/'
+        f"http://localhost:{PORT}{SWAGGER_URL}/api/swagger.json"
+    )
+
+    # Register blueprint at URL
+    app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)
+
     app.run(host='0.0.0.0', port=PORT)
 
 

arcor2/nodes/manager.py

@@ -29,7 +29,7 @@
 from arcor2 import rest
 from arcor2.nodes import builder
 
-BUILDER_URL = os.getenv("ARCOR2_BUILDER_URL", f"http://127.0.0.1:{builder.PORT}")
+BUILDER_URL = os.getenv("ARCOR2_BUILDER_URL", f"http://0.0.0.0:{builder.PORT}")
 
 PORT = 6790
 

arcor2/nodes/server.py

@@ -17,6 +17,7 @@
 from aiologger import Logger  # type: ignore
 from aiologger.levels import LogLevel  # type: ignore
 
+import arcor2
 from arcor2.source.logic import program_src
 from arcor2.source.object_types import new_object_type_source
 from arcor2.source import SourceException
@@ -50,6 +51,8 @@
 
 logger = Logger.with_default_handlers(name='server', formatter=hlp.aiologger_formatter(), level=LogLevel.DEBUG)
 
+MANAGER_URL = os.getenv("ARCOR2_EXECUTION_URL", f"ws://0.0.0.0:{MANAGER_PORT}")
+
 SCENE: Union[Scene, None] = None
 PROJECT: Union[Project, None] = None
 
@@ -137,7 +140,7 @@ async def project_manager_client() -> None:
         await logger.info("Attempting connection to manager...")
 
         try:
-            MANAGER_URL = os.getenv("ARCOR2_EXECUTION_URL", f"http://127.0.0.1:{MANAGER_PORT}")
+
             async with websockets.connect(MANAGER_URL) as manager_client:
 
                 await logger.info("Connected to manager.")
@@ -1473,6 +1476,8 @@ def main():
 
     parser.add_argument("-v", "--verbose", help="Increase output verbosity",
                         action="store_const", const=LogLevel.DEBUG, default=LogLevel.INFO)
+    parser.add_argument('--version', action='version',
+                        version='{}'.format(arcor2.version()))
 
     args = parser.parse_args()
     logger.level = args.verbose

arcor2/nodes/tests/test_build.py

@@ -0,0 +1,7 @@
+from openapi_spec_validator import validate_spec  # type: ignore
+import yaml
+from subprocess import check_output
+
+
+def test_build_openapi():
+    validate_spec(yaml.full_load(check_output(["arcor2_builder", "--swagger"])))

arcor2/nodes/tests/test_server.py

@@ -5,6 +5,7 @@
 import json
 import time
 import uuid
+import logging
 
 import pytest  # type: ignore
 import websocket  # type: ignore
@@ -19,6 +20,8 @@
 
 pytest_plugins = ["docker_compose"]
 
+LOGGER = logging.getLogger(__name__)
+
 
 @pytest.fixture()
 def ws_client(request, function_scoped_container_getter):
@@ -29,6 +32,8 @@ def ws_client(request, function_scoped_container_getter):
     my_env = os.environ.copy()
     my_env["ARCOR2_PERSISTENT_STORAGE_URL"] = api_url.strip("/")
 
+    LOGGER.info(f"Storage URL: {api_url}")
+
     processes = []
 
     for cmd in ("arcor2_manager", "arcor2_server"):
@@ -39,7 +44,7 @@ def ws_client(request, function_scoped_container_getter):
 
     for _ in range(20):
         try:
-            ws.connect("ws://localhost:6789", timeout=1)
+            ws.connect("ws://0.0.0.0:6789", timeout=1)
             break
         except ConnectionRefusedError:
             time.sleep(1.0)
@@ -90,7 +95,7 @@ def test_object_types(ws_client):
             assert ot.abstract
 
         ws_client.send(rpc.GetActionsRequest(id=uuid.uuid4().int, args=rpc.TypeArgs(ot.type)).to_json())
-        actions = rpc.GetActionsResponse.from_json(ws_client.recv())
+        rpc.GetActionsResponse.from_json(ws_client.recv())
         # TODO do some tests regarding actions?
 
 

arcor2/persistent_storage.py

@@ -7,7 +7,7 @@
 from arcor2.exceptions import Arcor2Exception
 from arcor2 import rest
 
-URL = os.getenv("ARCOR2_PERSISTENT_STORAGE_URL", "http://127.0.0.1:11000")
+URL = os.getenv("ARCOR2_PERSISTENT_STORAGE_URL", "http://0.0.0.0:11000")
 
 # TODO logger
 # TODO thread to poll changes? how to "detect" changes?

arcor2/rest.py

@@ -57,12 +57,12 @@ def _send(url: str, op: Callable, data: Optional[Union[JsonSchemaMixin, List[Jso
             for dd in data:
                 d.append(convert_keys(dd.to_dict(), snake_case_to_camel_case))
         else:
-            d = convert_keys(data.to_dict(), snake_case_to_camel_case)
+            d = convert_keys(data.to_dict(), snake_case_to_camel_case)  # type: ignore
     else:
-        d = {}
+        d = {}  # type: ignore
 
     if params:
-        params = convert_keys(params, snake_case_to_camel_case)
+        params = convert_keys(params, snake_case_to_camel_case)  # type: ignore
     else:
         params = {}
 
@@ -76,7 +76,7 @@ def _send(url: str, op: Callable, data: Optional[Union[JsonSchemaMixin, List[Jso
     handle_response(resp)
 
     if not get_response:
-        return
+        return None
 
     print(resp.text)
 
@@ -93,13 +93,13 @@ def post(url: str, data: JsonSchemaMixin, params: Optional[Dict] = None):
 
 def put(url: str, data: Optional[Union[JsonSchemaMixin, Sequence[JsonSchemaMixin]]] = None,
         params: Optional[Dict] = None, data_cls: Type[T] = None) -> T:
-    ret = _send(url, requests.put, data, params, get_response=data_cls is not None)
+    ret = _send(url, requests.put, data, params, get_response=data_cls is not None)  # type: ignore
 
     if not data_cls:
-        return None
+        return None  # type: ignore
 
     try:
-        return data_cls.from_dict(ret)
+        return data_cls.from_dict(ret)  # type: ignore
     except ValidationError as e:
         print(f'{data_cls.__name__}: validation error "{e}" while parsing "{data}".')
         raise RestException("Invalid data.")
@@ -129,12 +129,12 @@ def get_data(url: str, body: Optional[JsonSchemaMixin] = None, params: Optional[
 def _get(url: str, body: Optional[JsonSchemaMixin] = None, params: Optional[Dict] = None) -> Any:
 
     if body is None:
-        body_dict = {}
+        body_dict = {}  # type: ignore
     else:
         body_dict = body.to_dict()
 
     if params:
-        params = convert_keys(params, snake_case_to_camel_case)
+        params = convert_keys(params, snake_case_to_camel_case)  # type: ignore
     else:
         params = {}