diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..840366c109 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,71 @@ +name: Benchmark + +on: + pull_request: + branches: + - og-develop + +permissions: + # deployments permission to deploy GitHub pages website + deployments: write + # contents permission to update benchmark contents in gh-pages branch + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +jobs: + profiling: + runs-on: [self-hosted, linux, gpu] + container: + image: stanfordvl/omnigibson-dev:latest + options: --gpus=all --privileged --user=root + env: + DISPLAY: "" + OMNIGIBSON_HEADLESS: 1 + volumes: + - /scr/omni-data/datasets:/data + - /usr/share/vulkan/icd.d/nvidia_icd.json:/etc/vulkan/icd.d/nvidia_icd.json + - /usr/share/vulkan/icd.d/nvidia_layers.json:/etc/vulkan/implicit_layer.d/nvidia_layers.json + - /usr/share/glvnd/egl_vendor.d/10_nvidia.json:/usr/share/glvnd/egl_vendor.d/10_nvidia.json + - /scr/omni-data/isaac-sim/cache/ov:/root/.cache/ov:rw + - /scr/omni-data/isaac-sim/cache/pip:/root/.cache/pip:rw + - /scr/omni-data/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw + - /scr/omni-data/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw + - /scr/omni-data/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw + - /scr/omni-data/isaac-sim/config:/root/.nvidia-omniverse/config:rw + - /scr/omni-data/isaac-sim/data:/root/.local/share/ov/data:rw + - /scr/omni-data/isaac-sim/documents:/root/Documents:rw + + defaults: + run: + shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0} + + steps: + - name: Fix home + run: echo "HOME=/root" >> $GITHUB_ENV + + - name: Checkout source + uses: actions/checkout@v3 + + - name: Install dev requirements + run: pip install -r requirements-dev.txt + + - name: Install + run: pip install -e . + + - name: Run performance benchmark + run: source /isaac-sim/setup_conda_env.sh && bash scripts/benchmark.sh + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'customSmallerIsBetter' + output-file-path: output.json + benchmark-data-dir-path: benchmark + fail-on-alert: true + alert-threshold: '200%' + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-on-alert: true + auto-push: true diff --git a/.github/workflows/examples-as-test.yml b/.github/workflows/examples-as-test.yml index ee47f6c2bf..1c819dc45f 100644 --- a/.github/workflows/examples-as-test.yml +++ b/.github/workflows/examples-as-test.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true path: omnigibson @@ -50,7 +50,7 @@ jobs: run: pip uninstall -y bddl - name: Checkout BDDL - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: repository: StanfordVL/bddl-dev ref: 581be50e7cfd2b3a1447aaa1b4fc2424b673339c diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64ed147f2a..c35794bf54 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,7 +19,7 @@ jobs: run: echo "HOME=/root" >> $GITHUB_ENV - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: submodules: true path: omnigibson-src diff --git a/omnigibson/utils/profiling_utils.py b/omnigibson/utils/profiling_utils.py new file mode 100644 index 0000000000..2fecd4a892 --- /dev/null +++ b/omnigibson/utils/profiling_utils.py @@ -0,0 +1,56 @@ +import gym +import omnigibson as og + +from time import time +from omnigibson.envs.env_base import Environment + +PROFILING_FIELDS = ["total time", "physics time", "render time", "non physics time", "get observation time", "task time", "action time"] + +class ProfilingEnv(Environment): + def step(self, action): + start = time() + # If the action is not a dictionary, convert into a dictionary + if not isinstance(action, dict) and not isinstance(action, gym.spaces.Dict): + action_dict = dict() + idx = 0 + for robot in self.robots: + action_dim = robot.action_dim + action_dict[robot.name] = action[idx: idx + action_dim] + idx += action_dim + else: + # Our inputted action is the action dictionary + action_dict = action + + # Iterate over all robots and apply actions + for robot in self.robots: + robot.apply_action(action_dict[robot.name]) + action_end = time() + # Run simulation step + # Possibly force playing + for i in range(og.sim.n_physics_timesteps_per_render): + super(type(og.sim), og.sim).step(render=False) + physics_end = time() + og.sim.render() + render_end = time() + og.sim._non_physics_step() + non_physics_end = time() + # Grab observations + obs = self.get_obs() + obs_end = time() + # Grab reward, done, and info, and populate with internal info + reward, done, info = self.task.step(self, action) + self._populate_info(info) + + if done and self._automatic_reset: + # Add lost observation to our information dict, and reset + info["last_observation"] = obs + obs = self.reset() + + # Increment step + self._current_step += 1 + end = time() + ret = [end-start, physics_end-action_end, render_end-physics_end, non_physics_end-render_end, \ + obs_end-non_physics_end, end-obs_end, action_end-start] + if self._current_step % 100 == 0: + print("total time: {:.3f} ms, physics time: {:.3f} ms, render time: {:.3f} ms, non physics time: {:.3f} ms, get obs time: {:.3f} ms, task time: {:.3f} ms, action time: {:.3f} ms, ".format(*ret)) + return obs, reward, done, info, ret diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100644 index 0000000000..66a8de8dd4 --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,16 @@ +# 1st batch: baselines +python tests/benchmark/profiling.py # baseline (fastest config possible) +python tests/benchmark/profiling.py -s Rs_int # for vision research +python tests/benchmark/profiling.py -s Rs_int -r 1 # for basic robotics research +python tests/benchmark/profiling.py -s Rs_int -r 3 # for multi-agent research + +# 2nd batch: compare different scenes +python tests/benchmark/profiling.py -r 1 -s house_single_floor +python tests/benchmark/profiling.py -r 1 -s grocery_store_cafe +python tests/benchmark/profiling.py -r 1 -s Pomaria_0_garden + +# 3rd batch: OG non-physics features +python tests/benchmark/profiling.py -r 1 -s Rs_int -w # fluids (water) +python tests/benchmark/profiling.py -r 1 -s Rs_int -c # soft body (cloth) +python tests/benchmark/profiling.py -r 1 -s Rs_int -p # macro particle system (diced objects) +python tests/benchmark/profiling.py -r 1 -s Rs_int -w -c -p # everything \ No newline at end of file diff --git a/tests/benchmark/profiling.py b/tests/benchmark/profiling.py new file mode 100644 index 0000000000..5ebd6dc751 --- /dev/null +++ b/tests/benchmark/profiling.py @@ -0,0 +1,168 @@ +import os +import argparse +import json +import omnigibson as og +import numpy as np +import omnigibson.utils.transform_utils as T + +from omnigibson.macros import gm +from omnigibson.systems import get_system +from omnigibson.object_states import Covered +from omnigibson.utils.profiling_utils import ProfilingEnv +from omnigibson.utils.constants import PrimType + +parser = argparse.ArgumentParser() + +parser.add_argument("-r", "--robot", type=int, default=0) +parser.add_argument("-s", "--scene", default="") +parser.add_argument("-f", "--flatcache", action='store_true') +parser.add_argument("-c", "--softbody", action='store_true') +parser.add_argument("-w", "--fluids", action='store_true') +parser.add_argument("-p", "--macro_particle_system", action='store_true') + +PROFILING_FIELDS = ["Total frame time", "Physics step time", "Render step time", "Non-physics step time"] + + +SCENE_OFFSET = { + "": [0, 0], + "Rs_int": [0, 0], + "Pomaria_0_garden": [0.3, 0], + "grocery_store_cafe": [-3.5, 3.5], + "house_single_floor": [0, 0], +} + + +def main(): + args = parser.parse_args() + # Modify flatcache, pathtracing, GPU, and object state settings + assert not (args.flatcache and args.softbody), "Flatcache cannot be used with softbody at the same time" + gm.ENABLE_FLATCACHE = args.flatcache + gm.ENABLE_HQ_RENDERING = args.fluids + gm.ENABLE_OBJECT_STATES = True + gm.ENABLE_TRANSITION_RULES = True + gm.USE_GPU_DYNAMICS = True + + cfg = {} + if args.robot > 0: + cfg["robots"] = [] + for i in range(args.robot): + cfg["robots"].append({ + "type": "Fetch", + "obs_modalities": ["scan", "rgb", "depth"], + "action_type": "continuous", + "action_normalize": True, + "position": [-1.3 + 0.75 * i + SCENE_OFFSET[args.scene][0], 0.5 + SCENE_OFFSET[args.scene][1], 0], + "orientation": [0., 0., 0.7071, -0.7071] + }) + + if args.scene: + assert args.scene in SCENE_OFFSET, f"Scene {args.scene} not found in SCENE_OFFSET" + cfg["scene"] = { + "type": "InteractiveTraversableScene", + "scene_model": args.scene, + } + else: + cfg["scene"] = { + "type": "Scene", + } + + cfg["objects"] = [{ + "type": "DatasetObject", + "name": "table", + "category": "breakfast_table", + "model": "rjgmmy", + "scale": 0.75, + "position": [0.5 + SCENE_OFFSET[args.scene][0], -1 + SCENE_OFFSET[args.scene][1], 0.4], + "orientation": [0., 0., 0.7071, -0.7071] + }] + + if args.softbody: + cfg["objects"].append({ + "type": "DatasetObject", + "name": "shirt", + "category": "t_shirt", + "model": "kvidcx", + "prim_type": PrimType.CLOTH, + "scale": 0.01, + "position": [-0.4, -1, 1.5], + "orientation": [0.7071, 0., 0.7071, 0.], + }) + + cfg["objects"].extend([{ + "type": "DatasetObject", + "name": "apple", + "category": "apple", + "model": "agveuv", + "scale": 1.5, + "position": [0.5 + SCENE_OFFSET[args.scene][0], -1 + SCENE_OFFSET[args.scene][1], 0.6], + "abilities": {"diceable": {}} if args.macro_particle_system else {} + }, + { + "type": "DatasetObject", + "name": "knife", + "category": "table_knife", + "model": "lrdmpf", + "scale": 2.5, + "position": [0.5 + SCENE_OFFSET[args.scene][0], -1 + SCENE_OFFSET[args.scene][1], 0.8], + "orientation": T.euler2quat([-np.pi / 2, 0, 0]) + }]) + + env = ProfilingEnv(configs=cfg, action_timestep=1/60., physics_timestep=1/240.) + env.reset() + + apple = env.scene.object_registry("name", "apple") + table = env.scene.object_registry("name", "table") + knife = env.scene.object_registry("name", "knife") + knife.keep_still() + knife.set_position_orientation( + position=apple.get_position() + np.array([-0.15, 0.0, 0.2]), + orientation=T.euler2quat([-np.pi / 2, 0, 0]), + ) + if args.fluids: + table.states[Covered].set_value(get_system("stain"), True) # TODO: water is buggy for now, temporarily use stain instead + + output, results = [], [] + + # Update the simulator's viewer camera's pose so it points towards the robot + og.sim.viewer_camera.set_position([SCENE_OFFSET[args.scene][0], -3 + SCENE_OFFSET[args.scene][1], 1]) + + for i in range(500): + from IPython import embed; embed() + + if args.robot: + result = env.step(np.array([np.random.uniform(-0.3, 0.3, env.robots[i].action_dim) for i in range(args.robot)]).flatten())[4][:4] + else: + result = env.step(None)[4][:4] + results.append(result) + + results = np.array(results) + for i, title in enumerate(PROFILING_FIELDS): + field = f"{args.scene}" if args.scene else "Empty scene" + if args.robot: + field += f", with {args.robot} Fetch" + if args.softbody: + field += ", cloth" + if args.fluids: + field += ", fluids" + if args.macro_particle_system: + field += ", macro particles" + if args.flatcache: + field += ", flatcache on" + output.append({ + "name": field, + "unit": "fps", + "value": np.mean(results[-300:, i]) * 1000, + "extra": [title, title] + }) + + ret = [] + if os.path.exists("output.json"): + with open("output.json", "r") as f: + ret = json.load(f) + ret.extend(output) + with open("output.json", "w") as f: + json.dump(ret, f) + og.shutdown() + +if __name__ == "__main__": + main()