HRI Subtask Manager

docker/hri/chromadb.yaml

@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   chromadb:
     image: chromadb/chroma:latest

docker/hri/docker-compose.yml

@@ -0,0 +1,9 @@
+include:
+  # Runs speech and nlp nodes
+  - path: hri-ros.yaml
+
+  # Runs Faster-whisper
+  - path: stt.yaml
+
+  # Chroma db for embeddings
+  - path: chromadb.yaml

docker/hri/devices.yaml → docker/hri/hri-ros.yaml

@@ -11,6 +11,7 @@ services:
       dockerfile: docker/hri/Dockerfile.cpu
       args:
         BASE_IMAGE: roborregos/home2:cpu_base
+
     image: roborregos/home2:hri-cpu
     volumes:
       - ../../:/workspace/src
@@ -27,7 +28,7 @@ services:
     env_file:
       - .env
     tty: true
+    entrypoint: [ "bash", "-il", "-c"]
     command: [
-        "bash",
-        # "source /workspace/ws/devel/setup.bash && roslaunch hri recepcionist_laptop.launch",
+      "colcon build --symlink-install --packages-select task_manager frida_interfaces frida_constants speech nlp embeddings && source ~/.bashrc && ros2 launch speech hri_launch.py"
       ]

frida_constants/frida_constants/hri_constants.py

@@ -1,8 +1,15 @@
 SPEAK_SERVICE = "/speech/speak"
-HEAR_SERVICE = "/speech/STT"
+STT_SERVICE_NAME = "/speech/STT"
 KEYWORD_TOPIC = "/speech/kws"
 COMMAND_INTERPRETER_SERVICE = "/nlp/command_interpreter"
-DATA_EXTRACTOR_SERVICE = "/nlp/data_extractor"
+EXTRACT_DATA_SERVICE = "/nlp/data_extractor"
 SENTENCE_BUILDER_SERVICE = "/nlp/sentence_builder"
 ITEM_CATEGORIZATION_SERVICE = "/nlp/item_categorization"
 CONVESATION_SERVICE = "/nlp/conversation"
+GRAMMAR_SERVICE = "/nlp/grammar"
+
+ADD_ITEM_SERVICE = "/nlp/embeddings/add_item_service"
+REMOVE_ITEM_SERVICE = "/nlp/embeddings/remove_item_service"
+UPDATE_ITEM_SERVICE = "/nlp/embeddings/update_item_service"
+QUERY_ITEM_SERVICE = "/nlp/embeddings/query_item_service"
+BUILD_EMBEDDINGS_SERVICE = "/nlp/embeddings/build_embeddings_service"

frida_interfaces/hri/srv/CommandInterpreter.srv

@@ -1,3 +1,3 @@
-string text_heard
+string text
 ---
-frida_interfaces/CommandList commands
+frida_interfaces/Command[] commands

frida_interfaces/hri/srv/Grammar.srv

@@ -0,0 +1,3 @@
+string text
+---
+string corrected_text

hri/README.md

@@ -46,21 +46,13 @@ In addition, the following files are required:
 docker compose -f cuda.yaml build
 # or -> docker compose -f cpu.yaml build
 
-# Use devices compose (for audio I/O)
+# Build and run HRI containers
 # pwd -> home2/docker/hri
-docker compose -f devices.yaml up
+docker compose up
 
-# Build packages
-## Enter the container
+# Enter the container (this container has the ros 2 environment)
 docker exec -it home2-hri-cuda-devices bash
 
-# Enable non-root ownership of the workspace
-# pwd -> /workspace
-sudo chown -R $(id -u):$(id -g) .
-
-# pwd -> /workspace
-colcon build --symlink-install --packages-select frida_interfaces speech nlp
-source install/setup.bash
 ```
 
 ## Running the project
@@ -70,7 +62,10 @@ Most of the final commands will be executed using the docker compose file.
 However, some testing commands are the following:
 
 ```bash
-# Speech (Remember to start the stt docker before)
+# Launch HRI (includes speech, and nlp)
+ros2 launch speech hri_launch.py
+
+# Speech (Remember to start the stt docker before, this is done automatically if running the hri docker compose file)
 ros2 launch speech devices_launch.py
 
 ros2 topic pub /speech/speak_now --once std_msgs/msg/String "data: 'Go to the kitchen and grab cookies'"
@@ -81,6 +76,24 @@ ros2 launch nlp nlp_launch.py
 ros2 topic pub /speech/raw_command std_msgs/msg/String "data: Go to the kitchen and grab cookies" --once
 ```
 
+## Other useful commands
+
+Source the environment (this is automatically done in the .bashrc)
+```bash
+source /workspace/install/setup.bash
+```
+
+Build the hri packages (this is automatically done in `hri-ros.yaml` docker compose file)
+```bash
+colcon build --symlink-install --packages-select task_manager frida_interfaces frida_constants speech nlp embeddings
+```
+
+Enable file permissions for the current user, this is useful if there is a mismatch between the user in the container and the user in the host.
+```bash
+# pwd -> home2
+sudo chown -R $(id -u):$(id -g) .
+```
+
 ## Speech pipeline
 
 ### AudioCapturer.py

hri/packages/embeddings/CMakeLists.txt

@@ -19,7 +19,7 @@ install(
   FILES_MATCHING PATTERN "*.py"
 )
 
-install(DIRECTORY launch
+install(DIRECTORY launch config
  DESTINATION share/${PROJECT_NAME})
 file(GLOB PYTHON_SCRIPTS scripts/*.py)
 

hri/packages/embeddings/config/item_categorization.yaml

@@ -0,0 +1,10 @@
+item_categorization:
+  ros__parameters:
+    Embeddings_model: all-MiniLM-L12-v2
+    collections_built: 0
+
+    ADD_ITEM_SERVICE: REPLACE
+    REMOVE_ITEM_SERVICE: REPLACE
+    UPDATE_ITEM_SERVICE: REPLACE
+    QUERY_ITEM_SERVICE: REPLACE
+    BUILD_EMBEDDINGS_SERVICE: REPLACE

hri/packages/embeddings/launch/chroma_launch.py

@@ -1,8 +1,22 @@
+import os
+
+from ament_index_python.packages import get_package_share_directory
 from launch import LaunchDescription
 from launch_ros.actions import Node
 
+from frida_constants import ModuleNames, parse_ros_config
+
 
 def generate_launch_description():
+    item_categorization_config = parse_ros_config(
+        os.path.join(
+            get_package_share_directory("embeddings"),
+            "config",
+            "item_categorization.yaml",
+        ),
+        [ModuleNames.HRI.value],
+    )["item_categorization"]["ros__parameters"]
+
     return LaunchDescription(
         [
             Node(
@@ -11,7 +25,7 @@ def generate_launch_description():
                 name="embeddings",
                 output="screen",
                 parameters=[
-                    {"collections_built": 0}
+                    item_categorization_config
                 ],  # Default value (0 means not built)
             ),
         ]

hri/packages/embeddings/package.xml

@@ -12,6 +12,7 @@
   <depend>rclcpp</depend>
   <depend>rclpy</depend>
   <depend>frida_interfaces</depend>
+  <depend>frida_constants</depend>
 
   <test_depend>ament_lint_auto</test_depend>
   <test_depend>ament_lint_common</test_depend>

hri/packages/embeddings/scripts/item_categorization.py

@@ -1,17 +1,19 @@
 #!/usr/bin/env python3
 from pathlib import Path
-import pandas as pd
+
 import chromadb
-from chromadb.utils import embedding_functions
+import pandas as pd
 import rclpy
+from chromadb.utils import embedding_functions
 from rclpy.node import Node
 from rclpy.parameter import Parameter
+
 from frida_interfaces.srv import (
     AddItem,
+    BuildEmbeddings,
+    QueryItem,
     RemoveItem,
     UpdateItem,
-    QueryItem,
-    BuildEmbeddings,
 )
 
 
@@ -23,27 +25,53 @@ def __init__(self):
         self.declare_parameter("Embeddings_model", "all-MiniLM-L12-v2")
         self.declare_parameter("collections_built", 0)  # Default: 0 (not built)
 
+        # Parameters for services
+        self.declare_parameter("ADD_ITEM_SERVICE", "add_item")
+        self.declare_parameter("REMOVE_ITEM_SERVICE", "remove_item")
+        self.declare_parameter("UPDATE_ITEM_SERVICE", "update_item")
+        self.declare_parameter("QUERY_ITEM_SERVICE", "query_item")
+        self.declare_parameter("BUILD_EMBEDDINGS_SERVICE", "build_embeddings")
+
+        # Resolve parameters
         model_name_ = (
             self.get_parameter("Embeddings_model").get_parameter_value().string_value
         )
 
+        add_item_service = (
+            self.get_parameter("ADD_ITEM_SERVICE").get_parameter_value().string_value
+        )
+        remove_item_service = (
+            self.get_parameter("REMOVE_ITEM_SERVICE").get_parameter_value().string_value
+        )
+        update_item_service = (
+            self.get_parameter("UPDATE_ITEM_SERVICE").get_parameter_value().string_value
+        )
+        query_item_service = (
+            self.get_parameter("QUERY_ITEM_SERVICE").get_parameter_value().string_value
+        )
+        build_embeddings_service = (
+            self.get_parameter("BUILD_EMBEDDINGS_SERVICE")
+            .get_parameter_value()
+            .string_value
+        )
+
         # Initialize services
         self.add_item_service = self.create_service(
-            AddItem, "add_item", self.add_item_callback
+            AddItem, add_item_service, self.add_item_callback
         )
         self.remove_item_service = self.create_service(
-            RemoveItem, "remove_item", self.remove_item_callback
+            RemoveItem, remove_item_service, self.remove_item_callback
         )
         self.update_item_service = self.create_service(
-            UpdateItem, "update_item", self.update_item_callback
+            UpdateItem, update_item_service, self.update_item_callback
         )
         self.query_item_service = self.create_service(
-            QueryItem, "query_item", self.query_item_callback
+            QueryItem, query_item_service, self.query_item_callback
         )
 
         # Create the BuildEmbeddings service
         self.build_embeddings_service = self.create_service(
-            BuildEmbeddings, "build_embeddings", self.build_embeddings_callback
+            BuildEmbeddings, build_embeddings_service, self.build_embeddings_callback
         )
 
         # Initialize ChromaDB client

hri/packages/nlp/config/command_interpreter.yaml

@@ -4,4 +4,4 @@ command_interpreter:
     model: "llama3.2"
     speech_command_topic: "/speech/raw_command"
     publish_command_topic: "/task_manager/commands"
-    temperature: 1.0
+    temperature: 0.0

hri/packages/nlp/config/extract_data.yaml

@@ -1,5 +1,6 @@
-command_interpreter:
+extract_data:
   ros__parameters:
     base_url: "http://localhost:11434/v1"
     model: "llama3.2"
-    EXTRACT_DATA_SERVICE_NAME: "/extract_data"
+    EXTRACT_DATA_SERVICE: REPLACE
+    temperature: 0.0

hri/packages/nlp/config/stop_listener.yaml → hri/packages/nlp/config/llm_utils.yaml

@@ -1,6 +1,8 @@
-command_interpreter:
+llm_utils:
   ros__parameters:
     base_url: "http://localhost:11434/v1"
     model: "llama3.2"
     SPEECH_COMMAND_TOPIC_NAME: "/speech/raw_command"
     OUT_COMMAND_TOPIC_NAME: "/stop_following"
+    GRAMMAR_SERVICE: REPLACE
+    temperature: 0.0

hri/packages/nlp/launch/nlp_launch.py

@@ -1,14 +1,39 @@
+#!/usr/bin/env python3
+
 import os
 
 from ament_index_python.packages import get_package_share_directory
 from launch import LaunchDescription
 from launch_ros.actions import Node
 
+from frida_constants import ModuleNames, parse_ros_config
+
 
 def generate_launch_description():
-    command_interpreter_config = os.path.join(
-        get_package_share_directory("nlp"), "config", "command_interpreter.yaml"
-    )
+    command_interpreter_config = parse_ros_config(
+        os.path.join(
+            get_package_share_directory("nlp"), "config", "command_interpreter.yaml"
+        ),
+        [ModuleNames.HRI.value],
+    )["command_interpreter"]["ros__parameters"]
+
+    extract_data_config = parse_ros_config(
+        os.path.join(
+            get_package_share_directory("nlp"),
+            "config",
+            "extract_data.yaml",
+        ),
+        [ModuleNames.HRI.value],
+    )["extract_data"]["ros__parameters"]
+
+    llm_utils_config = parse_ros_config(
+        os.path.join(
+            get_package_share_directory("nlp"),
+            "config",
+            "llm_utils.yaml",
+        ),
+        [ModuleNames.HRI.value],
+    )["llm_utils"]["ros__parameters"]
 
     return LaunchDescription(
         [
@@ -20,5 +45,21 @@ def generate_launch_description():
                 emulate_tty=True,
                 parameters=[command_interpreter_config],
             ),
+            Node(
+                package="nlp",
+                executable="extract_data.py",
+                name="extract_data",
+                output="screen",
+                emulate_tty=True,
+                parameters=[extract_data_config],
+            ),
+            Node(
+                package="nlp",
+                executable="llm_utils.py",
+                name="llm_utils",
+                output="screen",
+                emulate_tty=True,
+                parameters=[llm_utils_config],
+            ),
         ]
     )

hri/packages/nlp/package.xml

@@ -13,6 +13,8 @@
   <depend>rclcpp</depend>
   <depend>rclpy</depend>
   <depend>frida_interfaces</depend>
+  <depend>frida_constants</depend>
+
 
   <test_depend>ament_lint_auto</test_depend>
   <test_depend>ament_lint_common</test_depend>