Skip to content

Commit

Permalink
Merge pull request #24 from small-thinking/further-refine
Browse files Browse the repository at this point in the history 
Further refine
  • Loading branch information
@yxjiang
yxjiang authored Feb 26, 2024
2 parents 9d57fa9 + 2c2758e commit 7b56979
Show file tree
Hide file tree
Showing 6 changed files with 223 additions and 32 deletions.
8 changes: 4 additions & 4 deletions README.cn.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
[![图片替代文本](./images/screen.png)](https://www.bilibili.com/video/BV1ub4y1T7Jt/?vd_source=08295b5b4b3c5ece73fb91e3a54d202a)

## 构建并启动Docker容器

0. 将 .env.template 更名为 .env 并添加 OPEN_API_KEY。
1. 安装 [Docker](https://docs.docker.com/get-docker/) 和 [Docker Compose](https://docs.docker.com/compose/install/)。
```bash
docker-compose up --build -d
Expand All @@ -15,13 +15,13 @@ docker-compose up --build -d

[![IMAGE ALT TEXT HERE](./images/novnc.png)]

1. 登录到docker容器。
2. 登录到docker容器。
如果您使用vscode，您可以安装 Dev Container 扩展并打开正在运行的容器。否则，您可以通过运行以下命令登录到docker容器：
```
docker exec -it mnln-ros_dev_env-1 /bin/bash
```

2. 启动ROS2模拟。
3. 启动ROS2模拟。

```bash
cd /home/small-thinking/mnlm/mnlm/robot/robot_arm_ws
Expand All @@ -35,7 +35,7 @@ colcon build --symlink-install ; source install/setup.bash ; ros2 launch robot_a

![IMAGE ALT TEXT HERE](./images/gazebo.png)

3. 在您的宿主机上。您可以运行命令以启动基于声音的UI。
4. 在您的宿主机上。您可以运行命令以启动基于声音的UI。

进入项目文件夹：
```bash
Expand Down
4 changes: 3 additions & 1 deletion mnlm/client/gpt_control/assistant.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ def create_assistant(
tool_signatures = [tool.get_signature() for tool in tools.values()]
assistant = client.beta.assistants.create(
name="Voice Robot Controller",
instructions="You have a brain and a robot arm, and you receive voice command from the human being, and respond accordingly.",
instructions="""
You have a brain and a robot arm, and you receive voice command from the human being, and respond accordingly.
""",
tools=tool_signatures,
model="gpt-3.5-turbo-1106",
)
Expand Down
2 changes: 1 addition & 1 deletion mnlm/client/gpt_control/robot_arm.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def translate_prompt_to_sequence(self, prompt: str) -> str:
instruction = f"""
Please convert the following oral comamnd to machine readable operation json (list of json blobs)
according to the API document.
An example output would be:
{{
operations: [
Expand Down
58 changes: 47 additions & 11 deletions mnlm/client/knowledge/command_bank.json
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
{
"reset": {
"reset or stand still": {
"operations": [
{
"operation": "move_all_servos",
"parameters": {"angles": [0, 0, 0, 0, 0, 0, 0], "time": 500}
}
]
},
"bow or bend": {
"operations": [
{
"operation": "move_all_servos",
"parameters": {"angles": [0, 60, 60, 85, 0, 0, 0], "time": 500}
}
]
},
"A simple dance": {
"operations": [
{
Expand All @@ -27,27 +35,55 @@
}
]
},
"A complex dance": {
"A complex dance looks like subject 3": {
"operations": [
{
"operation": "move_all_servos",
"parameters": {"angles": [0, 45, 60, 60, 0, 0, 0], "time": 500}
"parameters": {"angles": [-60, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_single_servo",
"parameters": {"id": "servo0", "angle": -60, "time": 500}
"operation": "move_all_servos",
"parameters": {"angles": [0, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_single_servo",
"parameters": {"id": "servo0", "angle": 60, "time": 500}
"operation": "move_all_servos",
"parameters": {"angles": [60, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_single_servo",
"parameters": {"id": "servo0", "angle": -60, "time": 500}
"operation": "move_all_servos",
"parameters": {"angles": [0, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_single_servo",
"parameters": {"id": "servo0", "angle": 60, "time": 500}
"operation": "move_all_servos",
"parameters": {"angles": [-60, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [0, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [60, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [0, 60, 60, 60, 0, 0, 0], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [-60, 60, 30, 0, 0, -10, -10], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [60, 60, 30, 0, 0, 10, 10], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [-60, 60, 30, 0, 0, -10, -10], "time": 500}
},
{
"operation": "move_all_servos",
"parameters": {"angles": [60, 60, 30, 0, 0, 10, 10], "time": 500}
},
{
"operation": "move_all_servos",
Expand Down
Binary file modified mnlm/client/knowledge/index/instructions.index
View file
Binary file not shown.
183 changes: 168 additions & 15 deletions mnlm/client/knowledge/index/instructions_data.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[
{
"instruction": "reset",
"instruction": "reset or stand still",
"operations": [
{
"operation": "move_all_servos",
Expand All @@ -19,6 +19,26 @@
}
]
},
{
"instruction": "bow or bend",
"operations": [
{
"operation": "move_all_servos",
"parameters": {
"angles": [
0,
60,
60,
85,
0,
0,
0
],
"time": 500
}
}
]
},
{
"instruction": "A simple dance",
"operations": [
Expand Down Expand Up @@ -71,14 +91,29 @@
]
},
{
"instruction": "A complex dance",
"instruction": "A complex dance looks like subject 3 (\u79d1\u76ee3)",
"operations": [
{
"operation": "move_all_servos",
"parameters": {
"angles": [
-60,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
0,
45,
60,
60,
60,
0,
Expand All @@ -89,34 +124,152 @@
}
},
{
"operation": "move_single_servo",
"operation": "move_all_servos",
"parameters": {
"id": "servo0",
"angle": -60,
"angles": [
60,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_single_servo",
"operation": "move_all_servos",
"parameters": {
"id": "servo0",
"angle": 60,
"angles": [
0,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_single_servo",
"operation": "move_all_servos",
"parameters": {
"id": "servo0",
"angle": -60,
"angles": [
-60,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_single_servo",
"operation": "move_all_servos",
"parameters": {
"id": "servo0",
"angle": 60,
"angles": [
0,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
60,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
0,
60,
60,
60,
0,
0,
0
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
-60,
60,
30,
0,
0,
-10,
-10
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
60,
60,
30,
0,
0,
10,
10
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
-60,
60,
30,
0,
0,
-10,
-10
],
"time": 500
}
},
{
"operation": "move_all_servos",
"parameters": {
"angles": [
60,
60,
30,
0,
0,
10,
10
],
"time": 500
}
},
Expand Down

0 comments on commit 7b56979

Please sign in to comment.