Add profiling scripts

.github/workflows/benchmark.yml

@@ -0,0 +1,76 @@
+name: Benchmark
+
+on: 
+  pull_request:
+    branches:
+        - og-develop
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: [self-hosted, linux, gpu]
+    container:
+      image: stanfordvl/omnigibson-dev:latest
+      options: --gpus=all --privileged --user=root
+      env:
+        DISPLAY: ""
+        OMNIGIBSON_HEADLESS: 1
+      volumes:
+        - /scr/omni-data/datasets:/data
+        - /usr/share/vulkan/icd.d/nvidia_icd.json:/etc/vulkan/icd.d/nvidia_icd.json
+        - /usr/share/vulkan/icd.d/nvidia_layers.json:/etc/vulkan/implicit_layer.d/nvidia_layers.json
+        - /usr/share/glvnd/egl_vendor.d/10_nvidia.json:/usr/share/glvnd/egl_vendor.d/10_nvidia.json
+        - /scr/omni-data/isaac-sim/cache/ov:/root/.cache/ov:rw
+        - /scr/omni-data/isaac-sim/cache/pip:/root/.cache/pip:rw
+        - /scr/omni-data/isaac-sim/cache/glcache:/root/.cache/nvidia/GLCache:rw
+        - /scr/omni-data/isaac-sim/cache/computecache:/root/.nv/ComputeCache:rw
+        - /scr/omni-data/isaac-sim/logs:/root/.nvidia-omniverse/logs:rw
+        - /scr/omni-data/isaac-sim/config:/root/.nvidia-omniverse/config:rw
+        - /scr/omni-data/isaac-sim/data:/root/.local/share/ov/data:rw
+        - /scr/omni-data/isaac-sim/documents:/root/Documents:rw
+
+    defaults:
+      run:
+        shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0}
+
+    steps:
+      - name: Fix home
+        run: echo "HOME=/root" >> $GITHUB_ENV
+
+      - name: Checkout source
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+          path: omnigibson-src
+
+      - name: Install dev requirements
+        working-directory: omnigibson-src
+        run: pip install -r requirements-dev.txt
+
+      - name: Install
+        working-directory: omnigibson-src
+        run: pip install -e .
+
+      - name: Run performance benchmark
+        working-directory: omnigibson-src
+        run: source /isaac-sim/setup_conda_env.sh && source tests/benchmark.sh
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'customBiggerIsBetter'
+          output-file-path: output.json
+          fail-on-alert: true
+          alert-threshold: '200%'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          auto-push: true

omnigibson/examples/profiling/benchmark.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+import os
+import argparse
+import json
+import omnigibson as og
+import numpy as np
+
+from omnigibson.macros import gm
+from omnigibson.utils.profiling_utils import ProfilingEnv
+
+OUTPUT_DIR = os.path.join(os.path.expanduser("~"), "Desktop")
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("-r", "--robot", action='store_true')
+parser.add_argument("-s", "--scene", action='store_true')
+parser.add_argument("-f", "--flatcache", action='store_true')
+parser.add_argument("-p", "--particles", action='store_true')
+
+PROFILING_FIELDS = ["Total step fps", "Action generation fps", "Physics step fps", "Render step time", "Non physics step fps"]
+
+def main():
+    args = parser.parse_args()
+    # Modify flatcache, pathtracing, GPU, and object state settings
+    gm.ENABLE_FLATCACHE = args.flatcache
+    gm.ENABLE_HQ_RENDERING = False
+    gm.ENABLE_OBJECT_STATES = True
+    gm.ENABLE_TRANSITION_RULES = True
+    gm.ENABLE_GPU_DYNAMICS = True
+
+    cfg = {}
+    if args.robot:
+        cfg["robots"] =  [{
+            "type": "Fetch",
+            "obs_modalities": ["scan", "rgb", "depth"],
+            "action_type": "continuous",
+            "action_normalize": True,
+            "controller_config": {"arm_0": {"name": "JointController"}}
+        }]
+    if args.particles:
+        pass
+    if args.scene:
+        cfg["scene"] = {
+            "type": "InteractiveTraversableScene",
+            "scene_model": "Rs_int",
+        }
+    else:
+        cfg["scene"] = {
+            "type": "InteractiveTraversableScene",
+            "scene_model": "Rs_int",
+        }
+    env = ProfilingEnv(configs=cfg, action_timestep=1/60., physics_timestep=1/240.)
+    env.reset()
+
+    output, results = [], []
+    for i in range(500):
+        if args.robot:
+            result = env.step(np.random.uniform(-0.1, 0.1, env.robots[0].action_dim))[4][:5]
+        else:
+            result = env.step(None)[4][:5]
+        results.append(result)
+
+    results = np.array(results)
+    for i, field in enumerate(PROFILING_FIELDS):
+        field += " (Rs_int, " if args.scene else " (Rs_int, "
+        field += "with Fetch, " if args.robot else "without Fetch, "
+        field += "with particles & softbody " if args.flatcache else "without particles & softbody "
+        field += "flatcache on)" if args.flatcache else "flatcache off)"
+        output.append({
+            "name": field,
+            "unit": "fps",
+            "value": 1 / np.mean(results[-200:, i])
+        })
+
+    with open("output.json", "w") as f:
+        json.dump(output, f)
+    og.shutdown()
+
+if __name__ == "__main__":
+    main()

omnigibson/examples/profiling/speed.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+
+import os
+import omni
+import logging
+import argparse
+import omnigibson as og
+import json
+
+from omnigibson.macros import gm
+from omnigibson.utils.profiling_utils import *
+
+OUTPUT_DIR = os.path.join(os.path.expanduser("~"), "Desktop")
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("-p", "--particles", action='store_true')
+parser.add_argument("-s", "--scenes", action='store_true')
+parser.add_argument("-o", "--objects", action='store_true')
+parser.add_argument("-r", "--robots", action='store_true')
+
+
+def main():
+    args = parser.parse_args()
+    # Modify flatcache, pathtracing, GPU, and object state settings
+    gm.ENABLE_FLATCACHE = False
+    gm.ENABLE_HQ_RENDERING = False
+    gm.ENABLE_OBJECT_STATES = True
+    gm.ENABLE_TRANSITION_RULES = True
+    gm.SHOW_DISCLAIMERS = False
+    # Disable OmniGibson logging
+    log = omni.log.get_log()
+    log.enabled = False
+    og.log.setLevel(logging.FATAL)
+
+    env = ProfilingEnv(configs=dict(scene={"type": "Scene"}), action_timestep=1/60., physics_timestep=1/240.)
+    env.reset()
+
+    results = dict()
+    if args.objects:
+        results["objects"] = benchmark_objects(env)
+    if args.robots:
+        results["robots"] = benchmark_robots(env)
+    if args.scenes:  
+        results["scenes"] = benchmark_scenes(env)
+    if args.particles:
+        results["fluids"] = benchmark_particle_system(env) 
+
+    with open(os.path.join(OUTPUT_DIR, "benchmark_results.json"), 'w') as f:
+        json.dump(results, f)
+    plot_results(results, os.path.join(OUTPUT_DIR, "omnigibson_benchmark.pdf"))
+    og.shutdown()
+
+if __name__ == "__main__":
+    main()