blearn: Add files (#4)

* blearn: Add files

* readme: Add more context

.devcontainer/blearn/Dockerfile

@@ -0,0 +1,52 @@
+FROM nvcr.io/nvidia/pytorch:22.12-py3
+
+ARG TZ=Europe/Zurich
+ENV DEBIAN_FRONTEND=noninteractive
+ENV BLENDER_DIR=/home/asl/blender
+ARG VGLUSERS_GID=1004
+
+# Base packages
+COPY devcontainer_all_packages.sh /tmp/devcontainer_all_packages.sh
+RUN /tmp/devcontainer_all_packages.sh && rm -f /tmp/devcontainer_all_packages.sh
+
+ENV LANG="en_US.UTF-8" \
+    LANGUAGE="en_US:en" \
+    LC_ALL="en_US.UTF-8" \
+    BLENDER_USER_SCRIPTS=/home/asl/.config/blender/3.0/scripts/
+
+# Add user "asl" with sudo rights
+RUN groupadd -r asl && \
+    groupadd -g ${VGLUSERS_GID} vglusers && \
+    useradd --create-home --gid asl --groups dialout,plugdev,vglusers --shell /bin/bash asl && \
+    mkdir -p /etc/sudoers.d && \
+    echo 'asl ALL=NOPASSWD: ALL' > /etc/sudoers.d/asl
+
+# Blender: Compile with patch that allows rendering depth in background
+RUN apt-get --assume-yes install xpra subversion cmake libx11-dev libxxf86vm-dev libxcursor-dev libxi-dev libxrandr-dev libxinerama-dev libglew-dev
+RUN mkdir -p $BLENDER_DIR/lib && cd $BLENDER_DIR/lib && svn checkout https://svn.blender.org/svnroot/bf-blender/tags/blender-3.0-release/lib/linux_centos7_x86_64/
+RUN mkdir -p $BLENDER_DIR && cd $BLENDER_DIR && git clone https://github.com/marcojob/blender.git && cd blender && git checkout v3_0_1_background_patch && git submodule update --init --recursive
+RUN cd $BLENDER_DIR/blender && make
+RUN cd $BLENDER_DIR/blender && cmake ../build_linux -D WITH_CYCLES_CUDA_BINARIES=ON && make
+RUN echo "alias blender='$BLENDER_DIR/build_linux/bin/blender'" >> /home/asl/.bash_aliases
+
+# Blender: Install python deps
+RUN $BLENDER_DIR/build_linux/bin/3.0/python/bin/python3.9 -m ensurepip
+RUN $BLENDER_DIR/build_linux/bin/3.0/python/bin/python3.9 -m pip install pyyaml matplotlib scipy
+
+# Blender: Copy user preferences to enable CUDA (ideally this would work with bpy, not figured out)
+COPY userpref.blend /home/asl/.config/blender/3.0/config/
+
+# Blender addon: Download and install BlenderGIS
+RUN wget https://github.com/domlysz/BlenderGIS/archive/refs/tags/2210.zip -O /tmp/BlenderGIS-2210.zip && \
+    unzip /tmp/BlenderGIS-2210.zip -d /tmp/ && \
+    mkdir -p $BLENDER_USER_SCRIPTS/addons && \
+    mv /tmp/BlenderGIS-2210 $BLENDER_USER_SCRIPTS/addons/BlenderGIS && \
+    $BLENDER_DIR/build_linux/bin/blender -b --python-expr "import bpy; bpy.ops.preferences.addon_enable(module='BlenderGIS'); bpy.ops.wm.save_userpref()"
+
+# Git-prompt: Source
+RUN echo 'if [ -f "$HOME/.bash-git-prompt/gitprompt.sh" ]; then GIT_PROMPT_ONLY_IN_REPO=1; source "$HOME/.bash-git-prompt/gitprompt.sh"; fi' >> /home/asl/.bash_aliases
+
+# Clean up
+RUN rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/* /var/cache/apt/archives/*
+
+ENTRYPOINT ["/bin/bash"]

.devcontainer/blearn/devcontainer.json

@@ -0,0 +1,54 @@
+{
+    "image": "omavteam/radarmeetsvision:blearn-latest",
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "ms-python.python"
+            ],
+            "settings": {
+                "files.hotExit": "off",
+                "window.restoreWindows": "none",
+                "workbench.startupEditor": "none"
+            }
+        }
+    },
+    "remoteEnv": {
+        "DISPLAY": "${localEnv:DISPLAY}"
+    },
+    "remoteUser": "asl",
+    "initializeCommand": ".devcontainer/devcontainer_optional_mounts.sh",
+    "postStartCommand": "",
+    "mounts": [
+        {
+            "source": "${localEnv:HOME}/.bash-git-prompt",
+            "target": "/home/asl/.bash-git-prompt",
+            "type": "bind"
+        },
+        {
+            "source": "${localEnv:HOME}/Downloads",
+            "target": "/home/asl/Downloads",
+            "type": "bind"
+        },
+        {
+            "source": "${localEnv:HOME}/Storage",
+            "target": "/home/asl/Storage",
+            "type": "bind"
+        },
+        {
+            "source": "${localEnv:HOME}/.cache",
+            "target": "/home/asl/.cache",
+            "type": "bind"
+        }
+    ],
+    "runArgs": [
+        "--gpus",
+        "all",
+        "--privileged",
+        "--device=/dev/dri:/dev/dri",
+        "-v",
+        "/tmp/.X11-unix:/tmp/.X11-unix",
+        "-v",
+        "/dev/bus/usb:/dev/bus/usb",
+        "--net=host"
+    ]
+}

.devcontainer/build_and_push_images.sh

@@ -1 +1,2 @@
+docker buildx build -t omavteam/radarmeetsvision:blearn-latest -f blearn/Dockerfile --push .
 docker buildx build -t omavteam/radarmeetsvision:latest -f desktop/Dockerfile --push .

.devcontainer/desktop/Dockerfile

@@ -8,8 +8,8 @@ ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/hpcx/ompi/lib"
 ENV PATH="${PATH}:/opt/hpcx/ompi/bin"
 
 # All apt packages
-COPY devcontainer-all-packages.sh /tmp/devcontainer-all-packages.sh
-RUN /tmp/devcontainer-all-packages.sh && rm -f /tmp/devcontainer-all-packages.sh
+COPY devcontainer_all_packages.sh /tmp/devcontainer_all_packages.sh
+RUN /tmp/devcontainer_all_packages.sh && rm -f /tmp/devcontainer_all_packages.sh
 
 ENV LANG="en_US.UTF-8" \
     LANGUAGE="en_US:en" \

.devcontainer/desktop/devcontainer.json

@@ -16,7 +16,7 @@
         "DISPLAY": "${localEnv:DISPLAY}"
     },
     "remoteUser": "asl",
-    "initializeCommand": ".devcontainer/devcontainer-optional-mounts.sh",
+    "initializeCommand": ".devcontainer/devcontainer_optional_mounts.sh",
     "postStartCommand": "pip install -e .",
     "mounts": [
         {

README.md

@@ -1,9 +1,44 @@
 # radarmeetsvision
+This repository contains the code of the ICRA 2025 submission (in review) "Radar Meets Vision: Robustifying Monocular Metric Depth Prediction for Mobile Robotics". The submitted version can be found on [arxiv](https://arxiv.org/abs/2410.00736).
 
-## Overview
+## Setup
+The easiest way to get started is using the respective devcontainer.
+The devcontainer all dependencies installed, including a patch that allows rendering depth images without a screen attached to the host.
+To use the devcontainer, install Visual Studio Code and follow the [instructions](https://code.visualstudio.com/docs/devcontainers/containers).
+In short, only vscode and docker are required.
+If installing directly on the host machine is more desirable, the Dockerfiles are a good starting point for the required dependencies.
+
+## radarmeetsvision
+This directory contains the network in a python package format. It can be installed using `pip install .` .
+
+### Interface
+The file `interface.py` aims to provide an interface to the network.
+In this file, most methods to train and evaluate are provided.
+In the `scripts` directory, usage examples are provided.
+To reproduce the training and evaluation results, the two scripts `train.py` and `evaluate.py` can be run.
+For details on the process, we refer to the paper, but in general, no adjustements are needed.
 
 ### metric_depth_network
-Contains the metric network based on [Depth Anything V2](https://github.com/DepthAnything/Depth-Anything-V2). All important code diffs compared to this upstream codebase are outlined in general.
+Contains the metric network based on [Depth Anything V2](https://github.com/DepthAnything/Depth-Anything-V2).
+All important code diffs compared to this upstream codebase are outlined in general.
+
+### Pretrained networks
+We provide the networks obtained from training and used for evaluation in the paper [here](tbd).
+
+## Training datasets
+**blearn** is a tool that allows you to generate a synthetic image and depth training dataset using Blender.
+Together with a mesh and texture obtained from photogrammetry, realistic synthetic datasets can be generated.
+The script is executed with Blender's built-in Python interpreter, which has the advantage that the Blender Python API is correctly loaded already.
+
+### Download existing datasets
+Existing datasets can be downloaded from [here](tdb). The download contains the datasets, as well as the blender projects used to obtain the datasets.
+
+### Generating training datasets
+In order to (re-)generate the training datasets, the following steps are needed:
+1. Obtaining .fbx mesh and texture for the area of interest. We use aerial photogrammetry and Pix4D to create this output.
+2. Import the fbx file into a new blender project. Ensure that only the following elements are present in the blender project: A mesh, a camera and a sun light source. As convenience, we provide the blender projects used in the paper.
+3. Create a configuration file: Easiest is to start from an existing configuration file. The main values to adjust are the camera intrinsics, as well as the extent of the mesh, i.e. `paths`: `xmin`, `xmax`, ... which defines the area which is sampled on the mesh and the number of samples the dataset should contain. One thing to consider is the position and orientation with respect to the blender origin. The z-coordinate specification in this tool is always with respect to the distance to ground, essentially doing terrain following. If you are using one of the provided meshes, then the extent does not need to be adjusted.
+4. The dataset rendering can then be started using: `blender -b <path to blender project file> --python blearn.py`. Ensure that you adjust the config file accordingly in `blearn.py`.
 
-## Unit tests
-Unittests can be run either using `python3 -m unittest` or the CI script `ci/pr_unittest.bash`.
+## Validation datasets
+The method for obtaining the validation datasets is described in the paper. The datasets are made available [here](tbd).

blearn/blearn.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+""" Main script to run blearn, generating a learning dataset in Blender"""
+
+
+__author__ = "Marco Job"
+
+import sys
+import os
+import bpy
+
+# Setup the path to find modules
+dn = os.path.dirname(__file__)
+
+if dn not in sys.path:
+    sys.path.append(dn)
+
+from src.blender import Blender
+
+def main():
+    b = Blender(config_file="config/config_rhone.yml")
+    b.start()
+
+
+if __name__ == "__main__":
+    main()

blearn/config/config.yml

@@ -0,0 +1,51 @@
+camera:
+  f: 3.5
+  sx: 4.96
+  sy: 3.72
+sun:
+  energy_min: 10.0
+  energy_max: 10.0
+  local_time_min: 10.0
+  local_time_max: 16.0
+output:
+  dir: output
+  resolution_x: 640
+  resolution_y: 480
+  format: JPEG
+  render: CYCLES
+  use_second_view: True
+paths:
+  number_of_samples: 10000
+  x_min: 0.0
+  x_max: 600.0
+  y_min: 0.0
+  y_max: 250.0
+  z_min: 1.0
+  z_max: 51.0
+  euler_x_min: -0.1745
+  euler_x_max: 0.1745
+  euler_y_min: -0.1745
+  euler_y_max: 0.1745
+  euler_z_min: -3.141
+  euler_z_max: 3.141
+  seed: 1
+demo:
+  radius: 20.0
+  x: 200.0
+  y: 100.0
+  z: 30.0
+landing:
+  x: 351.1
+  y: 102.9
+  z_start: 51.0
+  z_end: 0.0
+  down_vel: 0.5
+  side_acc: 1.0
+  fps: 20
+  euler_x_min: -0.1745
+  euler_x_max: 0.1745
+  euler_y_min: -0.1745
+  euler_y_max: 0.1745
+  euler_z_min: -0.087
+  euler_z_max: 0.087
+  poses_file: /media/scratch2/mjob/poses.txt

blearn/config/config_mountain_area.yml

@@ -0,0 +1,31 @@
+mode: "training"
+camera:
+  f: 4.143903289
+  sx: 4.968
+  sy: 3.726
+sun:
+  energy_min: 10.0
+  energy_max: 10.0
+  local_time_min: 10.0
+  local_time_max: 16.0
+output:
+  dir: output
+  resolution_x: 640
+  resolution_y: 480
+  format: JPEG
+  render: CYCLES
+paths:
+  number_of_samples: 10000
+  x_min: 0.0
+  x_max: 1400.0
+  y_min: 0.0
+  y_max: 1000.0
+  z_min: 1.0
+  z_max: 51.0
+  euler_x_min: -0.3926991
+  euler_x_max: 0.3926991
+  euler_y_min: -0.3926991
+  euler_y_max: 0.3926991
+  euler_z_min: -3.141
+  euler_z_max: 3.141
+  seed: 1

blearn/config/config_mountain_area_demo.yml

@@ -0,0 +1,35 @@
+mode: "demo"
+camera:
+  f: 4.143903289
+  sx: 4.968
+  sy: 3.726
+sun:
+  energy_min: 10.0
+  energy_max: 10.0
+  local_time_min: 10.0
+  local_time_max: 16.0
+output:
+  dir: output
+  resolution_x: 640
+  resolution_y: 480
+  format: JPEG
+  render: CYCLES
+paths:
+  number_of_samples: 100
+  x_min: 0.0
+  x_max: 1400.0
+  y_min: 0.0
+  y_max: 1000.0
+  z_min: 1.0
+  z_max: 51.0
+  euler_x_min: -0.3926991
+  euler_x_max: 0.3926991
+  euler_y_min: -0.3926991
+  euler_y_max: 0.3926991
+  euler_z_min: -3.141
+  euler_z_max: 3.141
+  seed: 1
+  demo_x: 700.0
+  demo_y: 450.0
+  demo_z: 50.0
+  demo_radius: 50.0

blearn/config/config_rhone.yml

@@ -0,0 +1,32 @@
+camera:
+  f: 3.5
+  sx: 4.96
+  sy: 3.72
+sun:
+  energy_min: 10.0
+  energy_max: 10.0
+  local_time_min: 10.0
+  local_time_max: 16.0
+output:
+  dir: output
+  resolution_x: 640
+  resolution_y: 480
+  format: JPEG
+  render: CYCLES
+  use_second_view: False
+paths:
+  number_of_samples: 2000
+  x_min: 0.0
+  x_max: 200.0
+  y_min: 0.0
+  y_max: 150.0
+  z_min: 1.0
+  z_max: 51.0
+  euler_x_min: -0.1745
+  euler_x_max: 0.1745
+  euler_y_min: -0.1745
+  euler_y_max: 0.1745
+  euler_z_min: -3.141
+  euler_z_max: 3.141
+  seed: 1
+

blearn/config/config_rhone_demo.yml

@@ -0,0 +1,36 @@
+mode: "demo"
+camera:
+  f: 3.5
+  sx: 4.96
+  sy: 3.72
+sun:
+  energy_min: 10.0
+  energy_max: 10.0
+  local_time_min: 10.0
+  local_time_max: 16.0
+output:
+  dir: output
+  resolution_x: 640
+  resolution_y: 480
+  format: JPEG
+  render: CYCLES
+  use_second_view: False
+paths:
+  number_of_samples: 100
+  x_min: 0.0
+  x_max: 200.0
+  y_min: 0.0
+  y_max: 150.0
+  z_min: 1.0
+  z_max: 51.0
+  euler_x_min: -0.1745
+  euler_x_max: 0.1745
+  euler_y_min: -0.1745
+  euler_y_max: 0.1745
+  euler_z_min: -3.141
+  euler_z_max: 3.141
+  seed: 1
+  demo_x: 100.0
+  demo_y: 75.0
+  demo_z: 50.0
+  demo_radius: 50.0