Merge branch 'main' into hri

docker/hri/Dockerfile.cpu

@@ -19,5 +19,7 @@ RUN apt-get install -y python3-pyaudio
 RUN apt-get update && apt-get install -y alsa-base alsa-utils gstreamer-1.0
 RUN apt-get install -y libasound-dev portaudio19-dev libportaudio2 libportaudiocpp0 pulseaudio
 
+USER ros
+
 # RUN apt install -y espeak
 # RUN pip install -U openai-whisper

docker/hri/Dockerfile.cuda

@@ -19,5 +19,7 @@ RUN apt-get install -y python3-pyaudio
 RUN apt-get update && apt-get install -y alsa-base alsa-utils gstreamer-1.0
 RUN apt-get install -y libasound-dev portaudio19-dev libportaudio2 libportaudiocpp0 pulseaudio
 
+USER ros
+
 # RUN apt install -y espeak
 # RUN pip install -U openai-whisper

docker/hri/devices.yaml

@@ -18,7 +18,6 @@ services:
       - /tmp:/tmp
 
     network_mode: host
-    user: ${LOCAL_USER_ID}:${LOCAL_GROUP_ID}
     privileged: true
     devices:
       - /dev:/dev

docker/hri/setup.bash

@@ -22,7 +22,3 @@ sudo usermod -aG audio $USER # Make sure current user has access to audio resour
 sudo chmod 777 /dev/snd/* # Allow access to audio devices.
 
 echo "Finished hri setup configuration for docker."
-echo "Note: some of the docker compose files depend on the following environment variables. You may set them on the terminal running the docker compose command, in an .env file in the docker compose directory, or in your ~/.bashrc file:
-
-- export LOCAL_USER_ID=\$(id -u)
-- export LOCAL_GROUP_ID=\$(id -g)"

docs/Setup.md

@@ -10,6 +10,7 @@ To install pre-commit and the repo's hooks, run the following commands:
 
 ```bash
 pip install pre-commit
+# pwd -> home2
 pre-commit install
 ```
 

frida_interfaces/hri/action/Conversate.action

@@ -0,0 +1,9 @@
+#goal definition
+string request
+int32 wait
+---
+#result definition
+int32 success
+---
+#feedback
+string execution_state

frida_interfaces/hri/msg/Command.msg

@@ -0,0 +1,3 @@
+string action
+string characteristic
+string complement

frida_interfaces/hri/msg/CommandList.msg

@@ -0,0 +1 @@
+frida_interfaces/Command[] commands

hri/.gitignore

@@ -5,3 +5,4 @@
 *.user
 **/.vscode/
 **/__pychache__/
+**/models

hri/README.md

@@ -59,8 +59,8 @@ docker exec -it home2-hri-cuda-devices bash
 sudo chown -R $(id -u):$(id -g) .
 
 # pwd -> /workspace
-colcon build --symlink-install  --packages-select frida_interfaces speech
-
+colcon build --symlink-install --packages-select frida_interfaces speech nlp
+source install/setup.bash
 ```
 
 ## Running the project
@@ -70,7 +70,13 @@ Most of the final commands will be executed using the docker compose file.
 However, some testing commands are the following:
 
 ```bash
-ros2 run speech say.py
+# Speech
+ros2 launch speech devices_launch.py
 
 ros2 topic pub /speech/speak_now --once std_msgs/msg/String "data: 'Go to the kitchen and grab cookies'"
+
+# NLP
+ros2 launch nlp nlp_launch.py
+
+ros2 topic pub /speech/raw_command std_msgs/msg/String "data: Go to the kitchen and grab cookies" --once
 ```

hri/packages/nlp/CMakeLists.txt

@@ -0,0 +1,70 @@
+cmake_minimum_required(VERSION 3.8)
+project(nlp)
+
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic)
+endif()
+
+# find dependencies
+find_package(ament_cmake REQUIRED)
+find_package(ament_cmake_python REQUIRED)
+find_package(rclcpp REQUIRED)
+find_package(rclpy REQUIRED)
+find_package(frida_interfaces REQUIRED)
+
+install(DIRECTORY launch config
+ DESTINATION share/${PROJECT_NAME})
+
+# Add Cpp executable
+# add_executable(ignore_laser src/ignore_laser.cpp)
+# ament_target_dependencies(audio_common_msgs)
+
+# Install Cpp executables
+# install(TARGETS
+#  ignore_laser
+#  DESTINATION lib/${PROJECT_NAME})
+
+ament_python_install_package(${PROJECT_NAME})
+
+# Install the Python module
+install(
+  DIRECTORY scripts/
+  DESTINATION ${PYTHON_INSTALL_DIR}/${PROJECT_NAME}
+  FILES_MATCHING PATTERN "*.py"
+)
+
+# Install ONNX files
+# install(
+#   DIRECTORY assets/
+#   DESTINATION lib/${PROJECT_NAME}
+#   FILES_MATCHING PATTERN "*.onnx"
+# )
+
+# Install JSON files
+# install(
+#   DIRECTORY assets/
+#   DESTINATION lib/${PROJECT_NAME}
+#   FILES_MATCHING PATTERN "*.json"
+# )
+
+file(GLOB PYTHON_SCRIPTS scripts/*.py)
+
+# Install python scripts
+install(PROGRAMS
+  ${PYTHON_SCRIPTS}
+  DESTINATION lib/${PROJECT_NAME}
+)
+
+if(BUILD_TESTING)
+  find_package(ament_lint_auto REQUIRED)
+  # the following line skips the linter which checks for copyrights
+  # comment the line when a copyright and license is added to all source files
+  set(ament_cmake_copyright_FOUND TRUE)
+  # the following line skips cpplint (only works in a git repo)
+  # comment the line when this package is in a git repo and when
+  # a copyright and license is added to all source files
+  set(ament_cmake_cpplint_FOUND TRUE)
+  ament_lint_auto_find_test_dependencies()
+endif()
+
+ament_package()

hri/packages/nlp/config/command_interpreter.yaml

@@ -0,0 +1,7 @@
+command_interpreter:
+  ros__parameters:
+    base_url: "http://localhost:11434/v1"
+    model: "llama3.2"
+    speech_command_topic: "/speech/raw_command"
+    publish_command_topic: "/task_manager/commands"
+    temperature: 1.0

hri/packages/nlp/launch/nlp_launch.py

@@ -0,0 +1,24 @@
+import os
+
+from ament_index_python.packages import get_package_share_directory
+from launch import LaunchDescription
+from launch_ros.actions import Node
+
+
+def generate_launch_description():
+    command_interpreter_config = os.path.join(
+        get_package_share_directory("nlp"), "config", "command_interpreter.yaml"
+    )
+
+    return LaunchDescription(
+        [
+            Node(
+                package="nlp",
+                executable="command_interpreter.py",
+                name="command_interpreter",
+                output="screen",
+                emulate_tty=True,
+                parameters=[command_interpreter_config],
+            ),
+        ]
+    )

hri/packages/nlp/nlp/assets/prompts.py

@@ -0,0 +1,114 @@
+from datetime import datetime
+
+import pytz
+
+CURRENT_CONTEXT = """
+Today is {CURRENT_DATE}.
+Your name is FRIDA (Friendly robotic interactive domestic assistant), a domestic assistant developed by RoBorregos.
+RoBorregos is the representative Robotic team from Tec de Monterrey, Campus Monterrey. It has around 40 members.
+You compete in the Robocup@home competition. Last summer you competed in the Netherlands, at the international competition. Last March you competed in TMR, obtaining 2nd place in Mexico.
+"""
+
+SYSTEM_PROMPT_CI_V2 = """
+You will be presented with an instruction from a human. The instruction may skip details, contain grammar mistakes or be ambiguous. The instruction could also make no sense at all.
+
+Your task is to divide the provided instruction into small commands. The commands contain an action, and could contain an optional characteristic and complement. The commands should also be listed in the correct order such the initial instruction can be achieved. 
+
+You may break a given instruction into the following commands:
+
+1. [
+    action = "clarification" (call when the provided instruction is unclear, and you need more information to proceed).
+    complement = Explain what you need to know.
+    characteristic = ""
+]
+
+2. [
+    action = "remember" (store information for later use).
+    complement = The information to store.
+    characteristic = ""
+]
+
+3. [
+    action = "go" (move to a location).
+    complement = The location to move to. Specify "past location" to return to a location stored with the "remember" action. Other ONLY available options are: "living_room", "office", "kitchen" and "entrance".
+    characteristic = "" Does not need to be specified, always leave empty.
+]
+
+4. [
+    action = "find" (search for an object).
+    complement = The object to search for.
+    characteristic = ""
+]
+
+5. [
+    action = "pick" (grab an object).
+    complement = The object to grab. Can be either "bowl" or "cereal_box"
+    characteristic = ""
+]
+
+6. [
+    action = "place" (place an object on the table).
+    complement = The object to place. Can be either "bowl" or "cereal_box"
+    characteristic = ""
+]
+
+7. [
+    action = "approach" (move closer to a person).
+    complement = The person to approach.
+    characteristic = ""
+]
+
+8. [
+    action = "give" (hand an object to a person).
+    complement = The object to hand. Can be either "bowl" or "cereal_box".
+    characteristic = ""
+]
+
+9. [
+    action = "describe" (describe a person).
+    complement = ""
+    characteristic = ""
+]
+
+10. [
+    action = "speak" (answer back using voice. ONLY use this command when a question is asked).
+    complement = The textual response to the user's question. To answer a question, you can use the following context: [{CURRENT_CONTEXT}]
+    characteristic = "".
+]
+
+"""
+
+SYSTEM_PROMPT_CI_V_EXPO = """
+You will be presented with an instruction from a human. The instruction may skip details, contain grammar mistakes or be ambiguous. The instruction could also make no sense at all.
+
+Your task is to divide the provided instruction into small commands. The commands contain an action, and could contain an optional characteristic and complement. The commands should also be listed in the correct order such the initial instruction can be achieved. 
+
+You may break a given instruction into the following commands:
+
+1. [
+    action = "clarification" (call when the provided instruction is unclear, and you need more information to proceed).
+    complement = Explain what you need to know.
+    characteristic = ""
+]
+
+2. [
+    action = "describe" (describe a person, assume that this describes any person talking to the robot and that only one person can talk to it at the same time).
+    complement = ""
+    characteristic = ""
+]
+
+3. [
+    action = "speak" (answer back using voice. ONLY use this command when a question is asked).
+    complement = The textual response to the user's question. To answer a question, you can use the following context: [{CURRENT_CONTEXT}]
+    characteristic = "" or "describe". If describe is specified, the robot will say the description saved with the describe command.
+]
+
+"""
+
+
+def get_system_prompt_ci_v2():
+    timezone = pytz.timezone("America/Mexico_City")
+    current_date = datetime.now(timezone).strftime("%Y-%m-%d %H:%M:%S")
+    return SYSTEM_PROMPT_CI_V2.format(
+        CURRENT_CONTEXT=CURRENT_CONTEXT.format(CURRENT_DATE=current_date)
+    )

hri/packages/nlp/package.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="3">
+  <name>nlp</name>
+  <version>0.0.0</version>
+  <description>TODO: Package description</description>
+  <maintainer email="[email protected]">RoBorregos</maintainer>
+  <license>TODO: License declaration</license>
+
+  <buildtool_depend>ament_cmake</buildtool_depend>
+  <buildtool_depend>ament_cmake_python</buildtool_depend>
+
+  <depend>rclcpp</depend>
+  <depend>rclpy</depend>
+  <depend>frida_interfaces</depend>
+
+  <test_depend>ament_lint_auto</test_depend>
+  <test_depend>ament_lint_common</test_depend>
+
+  <export>
+    <build_type>ament_cmake</build_type>
+  </export>
+</package>