openvla intergration

README.md

@@ -1,4 +1,20 @@
-# SimplerEnv: Simulated Manipulation Policy Evaluation Environments for Real Robot Setups
+# SimplerEnv: Simulated Manipulation Policy Evaluation Environments for Real Robot Setups + OpenVLA policy
+
+> [!IMPORTANT]
+> This a fork repo from [SimplerEnv](https://github.com/DelinQu/SimplerEnv) with [openvla](https://github.com/openvla/openvla) policy intergration. The evaluation results are not very satisfactory, possibly due to my code implementation and the OpenVLA model itself. I need about one day to complete all the experiments and organize the data before report the results. If you have any questions, please feel free to suggest code modifications and raise issues.
+> * the evaluation scripts keep consistent with original repo under ./scripts/
+
+```bash
+openvla_bridge.sh
+openvla_drawer_variant_agg.sh
+openvla_drawer_visual_matching.sh
+openvla_move_near_variant_agg.sh
+openvla_move_near_visual_matching.sh
+openvla_pick_coke_can_variant_agg.sh
+openvla_pick_coke_can_visual_matching.sh
+openvla_put_in_drawer_variant_agg.sh
+openvla_put_in_drawer_visual_matching.sh
+```
 
 [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/simpler-env/SimplerEnv/blob/main/example.ipynb)
 

scripts/openvla_bridge.sh

@@ -0,0 +1,50 @@
+gpu_id=0
+policy_model=openvla
+ckpt_path="openvla/openvla-7b"
+
+scene_name=bridge_table_1_v1
+robot=widowx
+rgb_overlay_path=ManiSkill2_real2sim/data/real_inpainting/bridge_real_eval_1.png
+robot_init_x=0.147
+robot_init_y=0.028
+export DISPLAY=:1.0
+# VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/nvidia_icd.json
+# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
+#   --robot ${robot} --policy-setup widowx_bridge \
+#   --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
+#   --env-name PutCarrotOnPlateInScene-v0 --scene-name ${scene_name} \
+#   --rgb-overlay-path ${rgb_overlay_path} \
+#   --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
+#   --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
+#
+# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
+#   --robot ${robot} --policy-setup widowx_bridge \
+#   --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
+#   --env-name StackGreenCubeOnYellowCubeBakedTexInScene-v0 --scene-name ${scene_name} \
+#   --rgb-overlay-path ${rgb_overlay_path} \
+#   --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
+#   --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
+#
+# python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
+#   --robot ${robot} --policy-setup widowx_bridge \
+#   --control-freq 5 --sim-freq 500 --max-episode-steps 60 \
+#   --env-name PutSpoonOnTableClothInScene-v0 --scene-name ${scene_name} \
+#   --rgb-overlay-path ${rgb_overlay_path} \
+#   --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
+#   --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
+
+
+scene_name=bridge_table_1_v2
+robot=widowx_sink_camera_setup
+rgb_overlay_path=ManiSkill2_real2sim/data/real_inpainting/bridge_sink.png
+robot_init_x=0.127
+robot_init_y=0.06
+
+python simpler_env/main_inference.py --policy-model ${policy_model} --ckpt-path ${ckpt_path} \
+  --robot ${robot} --policy-setup widowx_bridge \
+  --control-freq 5 --sim-freq 500 --max-episode-steps 120 \
+  --env-name PutEggplantInBasketScene-v0 --scene-name ${scene_name} \
+  --rgb-overlay-path ${rgb_overlay_path} \
+  --robot-init-x ${robot_init_x} ${robot_init_x} 1 --robot-init-y ${robot_init_y} ${robot_init_y} 1 --obj-variation-mode episode --obj-episode-range 0 24 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1;
+

scripts/openvla_drawer_variant_agg.sh

@@ -0,0 +1,82 @@
+# shader_dir=rt means that we turn on ray-tracing rendering; this is quite crucial for the open / close drawer task as policies often rely on shadows to infer depth
+declare -a ckpt_paths=(
+"openvla/openvla-7b"
+)
+
+declare -a env_names=(
+OpenTopDrawerCustomInScene-v0
+OpenMiddleDrawerCustomInScene-v0
+OpenBottomDrawerCustomInScene-v0
+CloseTopDrawerCustomInScene-v0
+CloseMiddleDrawerCustomInScene-v0
+CloseBottomDrawerCustomInScene-v0
+)
+
+EXTRA_ARGS="--enable-raytracing"
+
+
+# base setup
+scene_name=frl_apartment_stage_simple
+
+EvalSim() {
+  echo ${ckpt_path} ${env_name}
+
+  python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+    --robot google_robot_static \
+    --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+    --env-name ${env_name} --scene-name ${scene_name} \
+    --robot-init-x 0.65 0.85 3 --robot-init-y -0.2 0.2 3 \
+    --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0.0 0.0 1 \
+    --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+    ${EXTRA_ARGS}
+}
+
+
+for ckpt_path in "${ckpt_paths[@]}"; do
+  for env_name in "${env_names[@]}"; do
+    EvalSim
+  done
+done
+
+
+# backgrounds
+
+declare -a scene_names=(
+"modern_bedroom_no_roof"
+"modern_office_no_roof"
+)
+
+for scene_name in "${scene_names[@]}"; do
+  for ckpt_path in "${ckpt_paths[@]}"; do
+    for env_name in "${env_names[@]}"; do
+      EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt"
+      EvalSim
+    done
+  done
+done
+
+
+# lightings
+scene_name=frl_apartment_stage_simple
+
+for ckpt_path in "${ckpt_paths[@]}"; do
+  for env_name in "${env_names[@]}"; do
+    EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt light_mode=brighter"
+    EvalSim
+    EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt light_mode=darker"
+    EvalSim
+  done
+done
+
+
+# new cabinets
+scene_name=frl_apartment_stage_simple
+
+for ckpt_path in "${ckpt_paths[@]}"; do
+  for env_name in "${env_names[@]}"; do
+    EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt station_name=mk_station2"
+    EvalSim
+    EXTRA_ARGS="--additional-env-build-kwargs shader_dir=rt station_name=mk_station3"
+    EvalSim
+  done
+done

scripts/openvla_drawer_visual_matching.sh

@@ -0,0 +1,132 @@
+# shader_dir=rt means that we turn on ray-tracing rendering; this is quite crucial for the open / close drawer task as policies often rely on shadows to infer depth
+declare -a ckpt_paths=(
+"openvla/openvla-7b"
+)
+
+declare -a env_names=(
+OpenTopDrawerCustomInScene-v0
+OpenMiddleDrawerCustomInScene-v0
+OpenBottomDrawerCustomInScene-v0
+CloseTopDrawerCustomInScene-v0
+CloseMiddleDrawerCustomInScene-v0
+CloseBottomDrawerCustomInScene-v0
+)
+
+# URDF variations
+declare -a urdf_version_arr=("recolor_cabinet_visual_matching_1" "recolor_tabletop_visual_matching_1" "recolor_tabletop_visual_matching_2" None)
+
+for urdf_version in "${urdf_version_arr[@]}"; do
+
+EXTRA_ARGS="--enable-raytracing --additional-env-build-kwargs station_name=mk_station_recolor light_mode=simple disable_bad_material=True urdf_version=${urdf_version}"
+
+EvalOverlay() {
+# A0
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.644 0.644 1 --robot-init-y -0.179 -0.179 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.03 -0.03 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a0.png \
+  ${EXTRA_ARGS}
+
+# A1
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.765 0.765 1 --robot-init-y -0.182 -0.182 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.02 -0.02 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a1.png \
+  ${EXTRA_ARGS}
+
+# A2
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.889 0.889 1 --robot-init-y -0.203 -0.203 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.06 -0.06 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_a2.png \
+  ${EXTRA_ARGS}
+
+# B0
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.652 0.652 1 --robot-init-y 0.009 0.009 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b0.png \
+  ${EXTRA_ARGS}
+
+# B1
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.752 0.752 1 --robot-init-y 0.009 0.009 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b1.png \
+  ${EXTRA_ARGS}
+
+# B2
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.851 0.851 1 --robot-init-y 0.035 0.035 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_b2.png \
+  ${EXTRA_ARGS}
+
+# C0
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.665 0.665 1 --robot-init-y 0.224 0.224 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 0 0 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c0.png \
+  ${EXTRA_ARGS}
+
+# C1
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.765 0.765 1 --robot-init-y 0.222 0.222 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.025 -0.025 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c1.png \
+  ${EXTRA_ARGS}
+
+# C2
+python simpler_env/main_inference.py --policy-model openvla --ckpt-path ${ckpt_path} \
+  --robot google_robot_static \
+  --control-freq 3 --sim-freq 513 --max-episode-steps 113 \
+  --env-name ${env_name} --scene-name dummy_drawer \
+  --robot-init-x 0.865 0.865 1 --robot-init-y 0.222 0.222 1 \
+  --robot-init-rot-quat-center 0 0 0 1 --robot-init-rot-rpy-range 0 0 1 0 0 1 -0.025 -0.025 1 \
+  --obj-init-x-range 0 0 1 --obj-init-y-range 0 0 1 \
+  --rgb-overlay-path ./ManiSkill2_real2sim/data/real_inpainting/open_drawer_c2.png \
+  ${EXTRA_ARGS}
+}
+
+
+for ckpt_path in "${ckpt_paths[@]}"; do
+  for env_name in "${env_names[@]}"; do
+    EvalOverlay
+  done
+done
+
+
+
+done