feat: added distributed docker compose files

una-auxme · Jun 28, 2024 · a890f26 · a890f26
1 parent a0f4533
commit a890f26
Show file tree

Hide file tree

Showing 5 changed files with 195 additions and 1 deletion.
diff --git a/build/Taskfile b/build/Taskfile
@@ -16,6 +16,16 @@ task:run_dev() {
   docker:compose -f docker-compose.dev.yaml up
 }
 
+task:run_distributed() {
+  xhost +local:docker
+  docker:compose -f docker-compose.distributed.yaml up
+}
+
+task:run_dev_distributed() {
+  xhost +local:docker
+  docker:compose -f docker-compose.dev.distributed.yaml up
+}
+
 task:restart() {
     container="$1"
     docker:compose restart "${container:-agent}"

diff --git a/build/docker-compose.dev.distributed.yaml b/build/docker-compose.dev.distributed.yaml
@@ -0,0 +1,77 @@
+version: "3"
+
+services:
+  flake8:
+    image: alpine/flake8
+    command: .
+    volumes:
+      - ../:/apps
+
+  comlipy:
+    build: docker/comlipy
+    command: .
+    volumes:
+      - ../:/apps
+
+  mdlint:
+    image: peterdavehello/markdownlint:0.32.2
+    command: markdownlint .
+    volumes:
+      - ../:/md
+
+  roscore:
+    image: ros:noetic
+    command: roscore
+    environment:
+      - ROS_MASTER_URI=http://roscore:11311
+      - ROS_HOSTNAME=roscore
+    expose:
+      - 11311
+    networks:
+      - ros
+
+  agent:
+    build:
+      dockerfile: build/docker/agent/Dockerfile
+      args:
+        - USER_UID=${DOCKER_HOST_UNIX_UID:-1000}
+        - USER_GID=${DOCKER_HOST_UNIX_GID:-1000}
+      context: ../
+    init: true
+    tty: true
+    shm_size: 2gb
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [ gpu ]
+    #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS"
+    command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch"
+    # command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=paf23-carla-simulator-1 --track=MAP"
+
+    logging:
+      driver: "local"
+    environment:
+      - DISPLAY
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=all
+      - ROS_MASTER_URI=http://roscore:11311
+      - CARLA_SIM_HOST=<carla-server-ip-address>
+      - ROS_HOSTNAME=agent
+      - XDG_RUNTIME_DIR=/tmp/runtime-carla
+    volumes:
+      - /tmp/.X11-unix:/tmp/.X11-unix
+      # if you change the volume here also change the copy command
+      # in ``build/docker/build/Dockerfile
+      - ../:/workspace
+      # mount git config for dvc
+      - ../.gitconfig:/home/carla/.gitconfig
+      - ../:/workspace/
+    networks:
+      - carla
+      - ros
+
+networks:
+  carla:
+  ros:
diff --git a/build/docker-compose.dev.yaml b/build/docker-compose.dev.yaml
@@ -28,11 +28,18 @@ services:
       resources:
         limits:
           memory: 16G
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [ gpu ]
     expose:
       - 2000
       - 2001
       - 2002
     environment:
+      - DISPLAY
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=all
       - XDG_RUNTIME_DIR=/tmp/runtime-carla
     networks:
       - carla
@@ -61,6 +68,12 @@ services:
     init: true
     tty: true
     shm_size: 2gb
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [ gpu ]
     #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS"
     command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch"
     # command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=paf23-carla-simulator-1 --track=MAP"
@@ -69,6 +82,8 @@ services:
       driver: "local"
     environment:
       - DISPLAY
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=all
       - ROS_MASTER_URI=http://roscore:11311
       - CARLA_SIM_HOST=carla-simulator
       - ROS_HOSTNAME=agent

diff --git a/build/docker-compose.distributed.yaml b/build/docker-compose.distributed.yaml
@@ -0,0 +1,77 @@
+version: "3"
+
+services:
+  flake8:
+    image: alpine/flake8
+    command: .
+    volumes:
+      - ../:/apps
+
+  comlipy:
+    build: docker/comlipy
+    command: .
+    volumes:
+      - ../:/apps
+
+  mdlint:
+    image: peterdavehello/markdownlint:0.32.2
+    command: markdownlint .
+    volumes:
+      - ../:/md
+
+  roscore:
+    image: ros:noetic
+    command: roscore
+    environment:
+      - ROS_MASTER_URI=http://roscore:11311
+      - ROS_HOSTNAME=roscore
+    expose:
+      - 11311
+    networks:
+      - ros
+
+  agent:
+    build:
+      dockerfile: build/docker/agent/Dockerfile
+      args:
+        - USER_UID=${DOCKER_HOST_UNIX_UID:-1000}
+        - USER_GID=${DOCKER_HOST_UNIX_GID:-1000}
+      context: ../
+    init: true
+    tty: true
+    shm_size: 2gb
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [ gpu ]
+    #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS"
+    #command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch"
+    command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=<carla-server-ip-address> --track=MAP"
+
+    logging:
+      driver: "local"
+    environment:
+      - DISPLAY
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=all
+      - ROS_MASTER_URI=http://roscore:11311
+      - CARLA_SIM_HOST=<carla-server-ip-address>
+      - ROS_HOSTNAME=agent
+      - XDG_RUNTIME_DIR=/tmp/runtime-carla
+    volumes:
+      - /tmp/.X11-unix:/tmp/.X11-unix
+      # if you change the volume here also change the copy command
+      # in ``build/docker/build/Dockerfile
+      - ../:/workspace
+      # mount git config for dvc
+      - ../.gitconfig:/home/carla/.gitconfig
+      - ../:/workspace/
+    networks:
+      - carla
+      - ros
+
+networks:
+  carla:
+  ros:
diff --git a/build/docker-compose.yml b/build/docker-compose.yml
@@ -28,11 +28,18 @@ services:
       resources:
         limits:
           memory: 16G
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [ gpu ]
     expose:
       - 2000
       - 2001
       - 2002
     environment:
+      - DISPLAY
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=all
       - XDG_RUNTIME_DIR=/tmp/runtime-carla
     networks:
       - carla
@@ -61,14 +68,22 @@ services:
     init: true
     tty: true
     shm_size: 2gb
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: [ gpu ]
     #command: bash -c "sleep 10 && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/opt/leaderboard/leaderboard/autoagents/npc_agent.py --host=carla-simulator --track=SENSORS"
     #command: bash -c "sleep 10 && roslaunch agent/launch/dev.launch"
-    command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=paf23-carla-simulator-1 --track=MAP"
+    command: bash -c "sleep 10 && sudo chown -R carla:carla ../code/ && sudo chmod -R a+w ../code/ && python3 /opt/leaderboard/leaderboard/leaderboard_evaluator.py --debug=0 --routes=/opt/leaderboard/data/routes_devtest.xml --agent=/workspace/code/agent/src/agent/agent.py --host=carla-simulator --track=MAP"
 
     logging:
       driver: "local"
     environment:
       - DISPLAY
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=all
       - ROS_MASTER_URI=http://roscore:11311
       - CARLA_SIM_HOST=carla-simulator
       - ROS_HOSTNAME=agent