From 15d0845bf1d105eb74ffdf13e60726532b6cbc49 Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Mon, 2 Dec 2019 15:33:31 +0100 Subject: [PATCH 1/9] dockerfile for base image --- .dockerignore | 2 ++ Dockerfile | 28 ++++++++++++++++++++++++++++ docker/docker-compose.yml | 39 +++++++++++++++++++++++++++++++++++++++ docker/start.sh | 7 +++++++ 4 files changed, 76 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 docker/docker-compose.yml create mode 100644 docker/start.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..68920887 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +Dockerfile +docker/docker-compose.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7dc800a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3 + +RUN cd \ + && git clone https://github.com/s-knibbs/dataclasses-jsonschema.git \ + && cd dataclasses-jsonschema \ + && pip install -e . + +RUN cd \ + && git clone https://github.com/capripot/wait-for.git \ + && cd wait-for \ + && git checkout http-tests \ + && cp wait-for /wait-for + +RUN pip install --upgrade setuptools PyYAML + +COPY . /root/arcor2 + + +RUN cd ~/arcor2 \ + && pip install -e . + +RUN ln -s /root/arcor2/docker/start.sh /start.sh + +RUN mkdir -p ~/project +ENV PYTHONPATH ~ +ENV ARCOR2_PROJECT_PATH /root/project + +expose 6789 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 00000000..7159451c --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3' + +services: + arcor: + image: arcor2_base:0.1.0 + container_name: arcor2 + command: ["/wait-for", "http://pickmaster-storage:11000/projects", "--","/bin/bash", "/start.sh"] + depends_on: + - pickmaster-storage + ports: + - "6789:6789" + environment: + - ARCOR2_PERSISTENT_STORAGE_URL=http://pickmaster-storage:11000 + pickmaster-storage: + image: kinalisoft/storage:0.3.1 + container_name: "pickmaster-storage" + ports: + - "11000-11001:11000-11001" + environment: + MONGODB_URL: pickmaster-mongodb:27017 + links: + - pickmaster-mongodb + depends_on: + - pickmaster-mongodb + + pickmaster-mongodb: + image: mongo:4.2.0-bionic + container_name: "pickmaster-mongodb" + environment: + - MONGO_DATA_DIR=/data/db + - MONGO_LOG_DIR=/dev/null + volumes: + - mongodb:/data/db + expose: + - "27017" + +volumes: + mongodb: + diff --git a/docker/start.sh b/docker/start.sh new file mode 100644 index 00000000..7fc791e5 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +cd /root/arcor2/arcor2/user_objects || exit +python upload.py +arcor2_manager & +sleep 5 +arcor2_server From 00c9ecbfabe78b977854d3c3e7b3030580541fa0 Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 16:07:35 +0100 Subject: [PATCH 2/9] manager URL imported fron env variable --- arcor2/nodes/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arcor2/nodes/server.py b/arcor2/nodes/server.py index bbc62c34..690380ea 100755 --- a/arcor2/nodes/server.py +++ b/arcor2/nodes/server.py @@ -134,7 +134,8 @@ async def project_manager_client() -> None: await logger.info("Attempting connection to manager...") try: - async with websockets.connect(f"ws://localhost:{MANAGER_PORT}") as manager_client: + MANAGER_URL = os.getenv("ARCOR2_EXECUTION_URL", f"http://127.0.0.1:{builder.PORT}") + async with websockets.connect(MANAGER_URL) as manager_client: await logger.info("Connected to manager.") From 17f4033ba23d421e2afbb6ee15c044e5edc5ff15 Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 16:09:32 +0100 Subject: [PATCH 3/9] Dockerfiles for each service --- Dockerfile-arserver | 7 +++++++ Dockerfile-base | 26 ++++++++++++++++++++++++++ Dockerfile-build | 4 ++++ Dockerfile-execution | 3 +++ docker/start-arserver.sh | 5 +++++ docker/start-build.sh | 3 +++ docker/start-execution.sh | 3 +++ 7 files changed, 51 insertions(+) create mode 100644 Dockerfile-arserver create mode 100644 Dockerfile-base create mode 100644 Dockerfile-build create mode 100644 Dockerfile-execution create mode 100644 docker/start-arserver.sh create mode 100644 docker/start-build.sh create mode 100644 docker/start-execution.sh diff --git a/Dockerfile-arserver b/Dockerfile-arserver new file mode 100644 index 00000000..9c0c08c5 --- /dev/null +++ b/Dockerfile-arserver @@ -0,0 +1,7 @@ +FROM arcor2_base:latest + +RUN ln -s /root/arcor2/docker/start-arserver.sh /start.sh + +expose 6789 + +CMD /start.sh diff --git a/Dockerfile-base b/Dockerfile-base new file mode 100644 index 00000000..e0ddf2f1 --- /dev/null +++ b/Dockerfile-base @@ -0,0 +1,26 @@ +FROM python:3 + +RUN cd \ + && git clone https://github.com/s-knibbs/dataclasses-jsonschema.git \ + && cd dataclasses-jsonschema \ + && pip install -e . + +RUN cd \ + && git clone https://github.com/capripot/wait-for.git \ + && cd wait-for \ + && git checkout http-tests \ + && cp wait-for /wait-for + +RUN pip install --upgrade setuptools PyYAML + +COPY . /root/arcor2/ + +RUN cd ~/arcor2 \ + && pip install -e . + +RUN mkdir -p ~/project +ENV PYTHONPATH ~ +ENV ARCOR2_PROJECT_PATH /root/project + + +CMD ["/bin/sh", "/start.sh"] diff --git a/Dockerfile-build b/Dockerfile-build new file mode 100644 index 00000000..e4907a32 --- /dev/null +++ b/Dockerfile-build @@ -0,0 +1,4 @@ +FROM arcor2_base:latest + +RUN ln -s /root/arcor2/docker/start-build.sh /start.sh + diff --git a/Dockerfile-execution b/Dockerfile-execution new file mode 100644 index 00000000..87c30394 --- /dev/null +++ b/Dockerfile-execution @@ -0,0 +1,3 @@ +FROM arcor2_base:latest + +RUN ln -s /root/arcor2/docker/start-execution.sh /start.sh diff --git a/docker/start-arserver.sh b/docker/start-arserver.sh new file mode 100644 index 00000000..766374b6 --- /dev/null +++ b/docker/start-arserver.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd /root/arcor2/arcor2/user_objects || exit +python upload.py +arcor2_server diff --git a/docker/start-build.sh b/docker/start-build.sh new file mode 100644 index 00000000..41287b8a --- /dev/null +++ b/docker/start-build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +arcor2_builder diff --git a/docker/start-execution.sh b/docker/start-execution.sh new file mode 100644 index 00000000..2f7ad74c --- /dev/null +++ b/docker/start-execution.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +arcor2_manager From 68e7ceb1800f77681b430b465c2ceb5cf3c85eb9 Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 16:10:52 +0100 Subject: [PATCH 4/9] removed original Dockerfile --- Dockerfile | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 7dc800a6..00000000 --- a/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM python:3 - -RUN cd \ - && git clone https://github.com/s-knibbs/dataclasses-jsonschema.git \ - && cd dataclasses-jsonschema \ - && pip install -e . - -RUN cd \ - && git clone https://github.com/capripot/wait-for.git \ - && cd wait-for \ - && git checkout http-tests \ - && cp wait-for /wait-for - -RUN pip install --upgrade setuptools PyYAML - -COPY . /root/arcor2 - - -RUN cd ~/arcor2 \ - && pip install -e . - -RUN ln -s /root/arcor2/docker/start.sh /start.sh - -RUN mkdir -p ~/project -ENV PYTHONPATH ~ -ENV ARCOR2_PROJECT_PATH /root/project - -expose 6789 From 9639848b9b51c54c7e4e2194464b51e61216af4b Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 16:12:13 +0100 Subject: [PATCH 5/9] typo --- arcor2/nodes/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arcor2/nodes/server.py b/arcor2/nodes/server.py index 690380ea..a91eacc1 100755 --- a/arcor2/nodes/server.py +++ b/arcor2/nodes/server.py @@ -134,7 +134,7 @@ async def project_manager_client() -> None: await logger.info("Attempting connection to manager...") try: - MANAGER_URL = os.getenv("ARCOR2_EXECUTION_URL", f"http://127.0.0.1:{builder.PORT}") + MANAGER_URL = os.getenv("ARCOR2_EXECUTION_URL", f"http://127.0.0.1:{MANAGER_PORT}") async with websockets.connect(MANAGER_URL) as manager_client: await logger.info("Connected to manager.") From 9c7f7e221320045f9dc6088ad2529ef491533cd1 Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 16:27:45 +0100 Subject: [PATCH 6/9] updated dockerfile --- arcor2/docker/docker-compose.yml | 35 -------------------------------- docker/docker-compose.yml | 14 ++++++++++--- 2 files changed, 11 insertions(+), 38 deletions(-) delete mode 100644 arcor2/docker/docker-compose.yml diff --git a/arcor2/docker/docker-compose.yml b/arcor2/docker/docker-compose.yml deleted file mode 100644 index f4547194..00000000 --- a/arcor2/docker/docker-compose.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: "3" -services: - - arcor2_storage: - image: kinalisoft/storage:0.6.1 - container_name: "arcor2_storage" - networks: - - arcor2 - ports: - - "11000-11001:11000-11001" - environment: - MONGODB_URL: mongodb:27017 - links: - - mongodb - depends_on: - - mongodb - - mongodb: - image: mongo:4.2.0-bionic - container_name: "mongodb" - environment: - - MONGO_DATA_DIR=/data/db - - MONGO_LOG_DIR=/dev/null -# volumes: -# - mongodb:/data/db - networks: - - arcor2 - expose: - - "27017" - -#volumes: -# mongodb: - -networks: - arcor2: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7159451c..c3666a72 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,15 +2,23 @@ version: '3' services: arcor: - image: arcor2_base:0.1.0 + image: arcor2_arserver:latest container_name: arcor2 - command: ["/wait-for", "http://pickmaster-storage:11000/projects", "--","/bin/bash", "/start.sh"] depends_on: - pickmaster-storage ports: - "6789:6789" environment: - - ARCOR2_PERSISTENT_STORAGE_URL=http://pickmaster-storage:11000 + - ARCOR2_PERSISTENT_STORAGE_URL=http://pickmaster-storage:11000 + - ARCOR2_EXECUTION_URL=ws://arcor2_execution:6790 + arcor2_build: + image: arcor2_build:latest + container_name: arcor2_build + arcor2_execution: + image: arcor2_execution:latest + container_name: arcor2_execution + environment: + - ARCOR2_BUILDER_URL=http://arcor2_build:5007 pickmaster-storage: image: kinalisoft/storage:0.3.1 container_name: "pickmaster-storage" From 0278df63efb59be19fd20a9c8332048ecd31040b Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 16:52:36 +0100 Subject: [PATCH 7/9] updated image names --- Dockerfile-arserver | 2 +- Dockerfile-build | 2 +- Dockerfile-execution | 2 +- docker/docker-compose.yml | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile-arserver b/Dockerfile-arserver index 9c0c08c5..ef54734a 100644 --- a/Dockerfile-arserver +++ b/Dockerfile-arserver @@ -1,4 +1,4 @@ -FROM arcor2_base:latest +FROM arcor2/arcor2_base:latest RUN ln -s /root/arcor2/docker/start-arserver.sh /start.sh diff --git a/Dockerfile-build b/Dockerfile-build index e4907a32..ff79c81f 100644 --- a/Dockerfile-build +++ b/Dockerfile-build @@ -1,4 +1,4 @@ -FROM arcor2_base:latest +FROM arcor2/arcor2_base:latest RUN ln -s /root/arcor2/docker/start-build.sh /start.sh diff --git a/Dockerfile-execution b/Dockerfile-execution index 87c30394..11891e93 100644 --- a/Dockerfile-execution +++ b/Dockerfile-execution @@ -1,3 +1,3 @@ -FROM arcor2_base:latest +FROM arcor2/arcor2_base:latest RUN ln -s /root/arcor2/docker/start-execution.sh /start.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c3666a72..d95117c2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: arcor: - image: arcor2_arserver:latest + image: arcor2/arcor2_arserver:latest container_name: arcor2 depends_on: - pickmaster-storage @@ -12,10 +12,10 @@ services: - ARCOR2_PERSISTENT_STORAGE_URL=http://pickmaster-storage:11000 - ARCOR2_EXECUTION_URL=ws://arcor2_execution:6790 arcor2_build: - image: arcor2_build:latest + image: arcor2/arcor2_build:latest container_name: arcor2_build arcor2_execution: - image: arcor2_execution:latest + image: arcor2/arcor2_execution:latest container_name: arcor2_execution environment: - ARCOR2_BUILDER_URL=http://arcor2_build:5007 From f2167bdf559a2c674fce703a09a4c21a83db331d Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Wed, 11 Dec 2019 17:16:50 +0100 Subject: [PATCH 8/9] added missing import --- arcor2/nodes/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/arcor2/nodes/server.py b/arcor2/nodes/server.py index 2b82bb7b..467aca32 100755 --- a/arcor2/nodes/server.py +++ b/arcor2/nodes/server.py @@ -10,6 +10,7 @@ get_type_hints, Type import uuid import argparse +import os import websockets from websockets.server import WebSocketServerProtocol From 44996baf60101346522f30a216c9445be8e5597e Mon Sep 17 00:00:00 2001 From: Michal Kapinus Date: Thu, 12 Dec 2019 09:33:06 +0100 Subject: [PATCH 9/9] updated dockerfiles --- Dockerfile-arserver | 2 +- Dockerfile-base | 5 ----- Dockerfile-build | 2 +- Dockerfile-execution | 2 +- docker/docker-compose.yml | 44 ++++++++++++++++++++------------------- docker/start.sh | 7 ------- 6 files changed, 26 insertions(+), 36 deletions(-) delete mode 100644 docker/start.sh diff --git a/Dockerfile-arserver b/Dockerfile-arserver index ef54734a..0c90cae2 100644 --- a/Dockerfile-arserver +++ b/Dockerfile-arserver @@ -1,4 +1,4 @@ -FROM arcor2/arcor2_base:latest +FROM arcor2/arcor2_base RUN ln -s /root/arcor2/docker/start-arserver.sh /start.sh diff --git a/Dockerfile-base b/Dockerfile-base index e0ddf2f1..dbe3d2c3 100644 --- a/Dockerfile-base +++ b/Dockerfile-base @@ -1,10 +1,5 @@ FROM python:3 -RUN cd \ - && git clone https://github.com/s-knibbs/dataclasses-jsonschema.git \ - && cd dataclasses-jsonschema \ - && pip install -e . - RUN cd \ && git clone https://github.com/capripot/wait-for.git \ && cd wait-for \ diff --git a/Dockerfile-build b/Dockerfile-build index ff79c81f..b0e1f875 100644 --- a/Dockerfile-build +++ b/Dockerfile-build @@ -1,4 +1,4 @@ -FROM arcor2/arcor2_base:latest +FROM arcor2/arcor2_base RUN ln -s /root/arcor2/docker/start-build.sh /start.sh diff --git a/Dockerfile-execution b/Dockerfile-execution index 11891e93..dd443570 100644 --- a/Dockerfile-execution +++ b/Dockerfile-execution @@ -1,3 +1,3 @@ -FROM arcor2/arcor2_base:latest +FROM arcor2/arcor2_base RUN ln -s /root/arcor2/docker/start-execution.sh /start.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d95117c2..b08bafc4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,39 +1,41 @@ version: '3' services: - arcor: - image: arcor2/arcor2_arserver:latest - container_name: arcor2 + arserver: + image: arcor2/arcor2_kinali + container_name: arserver depends_on: - - pickmaster-storage + - project + - build + - execution ports: - "6789:6789" environment: - - ARCOR2_PERSISTENT_STORAGE_URL=http://pickmaster-storage:11000 - - ARCOR2_EXECUTION_URL=ws://arcor2_execution:6790 - arcor2_build: - image: arcor2/arcor2_build:latest - container_name: arcor2_build - arcor2_execution: - image: arcor2/arcor2_execution:latest - container_name: arcor2_execution + - ARCOR2_PERSISTENT_STORAGE_URL=http://project:11000 + - ARCOR2_EXECUTION_URL=ws://execution:6790 + build: + image: arcor2/arcor2_build + container_name: build + execution: + image: arcor2/arcor2_execution + container_name: execution environment: - - ARCOR2_BUILDER_URL=http://arcor2_build:5007 - pickmaster-storage: - image: kinalisoft/storage:0.3.1 - container_name: "pickmaster-storage" + - ARCOR2_BUILDER_URL=http://build:5007 + project: + image: kinalisoft/storage:0.6.1 + container_name: "project" ports: - "11000-11001:11000-11001" environment: - MONGODB_URL: pickmaster-mongodb:27017 + MONGODB_URL: mongodb:27017 links: - - pickmaster-mongodb + - mongodb depends_on: - - pickmaster-mongodb + - mongodb - pickmaster-mongodb: + mongodb: image: mongo:4.2.0-bionic - container_name: "pickmaster-mongodb" + container_name: "mongodb" environment: - MONGO_DATA_DIR=/data/db - MONGO_LOG_DIR=/dev/null diff --git a/docker/start.sh b/docker/start.sh deleted file mode 100644 index 7fc791e5..00000000 --- a/docker/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -cd /root/arcor2/arcor2/user_objects || exit -python upload.py -arcor2_manager & -sleep 5 -arcor2_server