Merge remote-tracking branch 'upstream/main' into training_rl_paper

.github/workflows/push.yml

@@ -1,9 +1,4 @@
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-name: Test package quality
+name: Test Package Quality
 
 on: push
 
@@ -26,6 +21,11 @@ jobs:
     - name: Install package
       run: |
         pip install -e .
+    - name: Run Linting
+      run: |
+        pre-commit install
+        pre-commit autoupdate
+        pre-commit run
     - name: Unit tests
       run: |
         python -m pytest tests/

.pre-commit-config.yaml

@@ -0,0 +1,60 @@
+# Install the pre-commit hooks below with
+# 'pre-commit install'
+
+# Auto-update the version of the hooks with
+# 'pre-commit autoupdate'
+
+# Run the hooks on all files with
+# 'pre-commit run --all'
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: check-ast
+    -   id: check-yaml
+    -   id: check-toml
+    -   id: check-added-large-files
+        args: ['--maxkb=10000']
+    -   id: check-docstring-first
+    -   id: check-executables-have-shebangs
+    -   id: check-shebang-scripts-are-executable
+    -   id: check-merge-conflict
+    -   id: debug-statements
+    -   id: trailing-whitespace
+        exclude: (^.*.txt)
+    -   id: end-of-file-fixer
+        exclude: (^.*.txt)
+    -   id: double-quote-string-fixer
+
+-   repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort
+        args: ['--line-length=110']
+
+-   repo: https://github.com/pre-commit/mirrors-autopep8
+    rev: v2.0.4
+    hooks:
+    -   id: autopep8
+        name: autopep8-default
+        args: ['-i', '--ignore=C0301', '--max-line-length=1000']
+        exclude: (^tests/|^safe_control_gym/math_and_models/transformations.py)
+    -   id: autopep8
+        name: autopep8-tests
+        args: ['-i', '--ignore=C0301,E501,E201,E241,E127', '--max-line-length=1000']
+        files: (^tests/|^safe_control_gym/math_and_models/transformations.py)
+
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        name: flake8_default
+        args: ['--ignore=E501']
+        exclude: (^safe_control_gym/__init__.py|^tests/|^safe_control_gym/math_and_models/transformations.py)
+    -   id: flake8
+        name: flake8_tests
+        args: ['--ignore=E501,E201,E241,E127']
+        files: (^tests/|^safe_control_gym/math_and_models/transformations.py)
+        exclude: ^tests/test_build.py

README.md

@@ -40,7 +40,7 @@ git clone https://github.com/utiasDSL/safe-control-gym.git
 cd safe-control-gym
 ```
 
-### Option A (recommended): using `conda`
+### (optional) Create a `conda` environment
 
 Create and access a Python 3.10 environment using
 [`conda`](https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html)
@@ -50,29 +50,15 @@ conda create -n safe python=3.10
 conda activate safe
 ```
 
+### Install
+
 Install the `safe-control-gym` repository
 
 ```bash
 pip install --upgrade pip
 pip install -e .
 ```
 
-### Option B: using venv and poetry
-
-Create and access a Python 3.10 virtual environment using
-[`pyenv`](https://github.com/pyenv/pyenv) and
-[`venv`](https://docs.python.org/3/library/venv.html)
-
-```bash
-pyenv install 3.10
-pyenv local 3.10
-python3 -m venv safe
-source safe/bin/activate
-pip install --upgrade pip
-pip install poetry
-poetry install
-```
-
 #### Note
 
 You may need to separately install `gmp`, a dependency of `pycddlib`:
@@ -97,6 +83,80 @@ Overview of [`safe-control-gym`](https://arxiv.org/abs/2109.06325)'s API:
 
 <img src="figures/config.png" alt="config" width="800">
 
+## Getting Started
+
+Familiarize with APIs and environments with the scripts in [`examples/`](https://github.com/utiasDSL/safe-control-gym/tree/main/examples)
+
+### 3D Quadrotor Lemniscate Trajectory Tracking with PID
+
+```bash
+cd ./examples/   # Navigate to the examples folder
+python3 pid/pid_experiment.py \
+    --algo pid \
+    --task quadrotor \
+    --overrides \
+        ./pid/config_overrides/quadrotor_3D/quadrotor_3D_tracking.yaml
+```
+
+<img src="figures/systems.png" alt="systems" width="450"> <img src="figures/figure8.gif" alt="trajectory" width="350">
+
+### Cartpole Stabilization with LQR
+
+```bash
+cd ./examples/   # Navigate to the examples folder
+python3 lqr/lqr_experiment.py \
+    --algo lqr \
+    --task cartpole \
+    --overrides \
+        ./lqr/config_overrides/cartpole/cartpole_stabilization.yaml \
+        ./lqr/config_overrides/cartpole/lqr_cartpole_stabilization.yaml
+```
+
+### 2D Quadrotor Trajectory Tracking with PPO
+
+```bash
+cd ./examples/rl/   # Navigate to the RL examples folder
+python3 rl_experiment.py \
+    --algo ppo \
+    --task quadrotor \
+    --overrides \
+        ./config_overrides/quadrotor_2D/quadrotor_2D_track.yaml \
+        ./config_overrides/quadrotor_2D/ppo_quadrotor_2D.yaml \
+    --kv_overrides \
+        algo_config.training=False
+```
+
+### Verbose API Example
+
+```bash
+cd ./examples/   # Navigate to the examples folder
+python3 no_controller/verbose_api.py \
+    --task cartpole \
+    --overrides no_controller/verbose_api.yaml
+```
+
+<img src="figures/prints.png" al="prints" width="800">
+
+## List of Implemented Controllers
+
+- [PID](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/pid/pid.py)
+- [LQR](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/lqr/lqr.py)
+- [iLQR](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/lqr/ilqr.py)
+- [Linear MPC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/mpc/linear_mpc.py)
+- [GP-MPC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/mpc/gp_mpc.py)
+- [SAC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/sac/sac.py)
+- [PPO](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/ppo/ppo.py)
+- [DDPG](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/ddpg/ddpg.py)
+- [Safety Layer](https://github.com/utiasDSL/safe-control-gym/tree/main/safe_control_gym/controllers/safe_explorer)
+- [RARL](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/rarl/rarl.py)
+- [RAP](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/rarl/rap.py)
+
+## List of Implemented Safety Filters
+
+- [MPSC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/safety_filters/mpsc/linear_mpsc.py)
+- [CBF](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/safety_filters/cbf/cbf.py)
+- [Neural Network CBF](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/safety_filters/cbf/cbf_nn.py)
+
 ## Performance
 
 We compare the sample efficiency of `safe-control-gym` with the original [OpenAI Cartpole][001] and [PyBullet Gym's Inverted Pendulum][002], as well as [`gym-pybullet-drones`][003].
@@ -130,43 +190,18 @@ Note that the Bullet engine frequency reported for `safe-control-gym` is typical
 [004]: https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/envs/gym_control/cartpole.py
 [005]: https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/envs/gym_pybullet_drones/quadrotor.py
 
-## Getting Started
-
-Familiarize with APIs and environments with the scripts in [`examples/`](https://github.com/utiasDSL/safe-control-gym/tree/main/examples)
-
+## Run Tests and Linting
+Tests can be run locally by executing:
 ```bash
-cd ./examples/                                                                    # Navigate to the examples folder
-python3 pid/pid_experiment.py --algo pid --task quadrotor --overrides ./pid/config_overrides/quadrotor_3D/quadrotor_3D_tracking.yaml       # PID trajectory tracking with the 3D quadcopter
-python3 no_controller/verbose_api.py --task cartpole --overrides no_controller/verbose_api.yaml               #  Printout of the extended safe-control-gym APIs
+python3 -m pytest ./tests/  # Run all tests
 ```
 
-## Systems Variables and 2D Quadrotor Lemniscate Trajectory Tracking
-
-<img src="figures/systems.png" alt="systems" width="450"> <img src="figures/figure8.gif" alt="trajectory" width="350">
-
-## Verbose API Example
-
-<img src="figures/prints.png" al="prints" width="800">
-
-## List of Implemented Controllers
-
-- [PID](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/pid/pid.py)
-- [LQR](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/lqr/lqr.py)
-- [iLQR](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/lqr/ilqr.py)
-- [Linear MPC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/mpc/linear_mpc.py)
-- [GP-MPC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/mpc/gp_mpc.py)
-- [SAC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/sac/sac.py)
-- [PPO](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/ppo/ppo.py)
-- [DDPG](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/ddpg/ddpg.py)
-- [Safety Layer](https://github.com/utiasDSL/safe-control-gym/tree/main/safe_control_gym/controllers/safe_explorer)
-- [RARL](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/rarl/rarl.py)
-- [RAP](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/controllers/rarl/rap.py)
-
-## List of Implemented Safety Filters
-
-- [MPSC](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/safety_filters/mpsc/linear_mpsc.py)
-- [CBF](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/safety_filters/cbf/cbf.py)
-- [Neural Network CBF](https://github.com/utiasDSL/safe-control-gym/blob/main/safe_control_gym/safety_filters/cbf/cbf_nn.py)
+Linting can be run locally with:
+```bash
+pre-commit install  # Install the pre-commit hooks
+pre-commit autoupdate  # Auto-update the version of the hooks
+pre-commit run --all  # Run the hooks on all files
+```
 
 ## References
 

examples/cbf/cbf_experiment.py

@@ -4,12 +4,12 @@
 import shutil
 from functools import partial
 
-import numpy as np
 import matplotlib.pyplot as plt
+import numpy as np
 
 from safe_control_gym.experiments.base_experiment import BaseExperiment
-from safe_control_gym.utils.registration import make
 from safe_control_gym.utils.configuration import ConfigFactory
+from safe_control_gym.utils.registration import make
 
 
 def run(plot=True, training=True, n_episodes=1, n_steps=None, curr_path='.', save_data=False):

examples/lqr/lqr_experiment.py

@@ -2,15 +2,15 @@
 
 import os
 import pickle
-from functools import partial
 from collections import defaultdict
+from functools import partial
 
-import numpy as np
 import matplotlib.pyplot as plt
+import numpy as np
 from matplotlib.ticker import FormatStrFormatter
 
-from safe_control_gym.experiments.base_experiment import BaseExperiment
 from safe_control_gym.envs.benchmark_env import Task
+from safe_control_gym.experiments.base_experiment import BaseExperiment
 from safe_control_gym.utils.configuration import ConfigFactory
 from safe_control_gym.utils.registration import make
 

examples/mpsc/config_overrides/cartpole/cartpole_stab.yaml

@@ -4,13 +4,13 @@ task_config:
   ctrl_freq: 15
   pyb_freq: 750
   physics: pyb
-  
+
   # state initialization
   init_state:
     init_x: 0.1
     init_x_dot: -1.5
     init_theta: -0.175
-    init_theta_dot: 0.5  
+    init_theta_dot: 0.5
   randomized_init: True
   randomized_inertial_prop: False
   normalized_rl_action_space: True

examples/mpsc/config_overrides/cartpole/cartpole_track.yaml

@@ -4,13 +4,13 @@ task_config:
   ctrl_freq: 15
   pyb_freq: 750
   physics: pyb
-  
+
   # state initialization
   init_state:
     init_x: 0
     init_x_dot: 0
     init_theta: 0
-    init_theta_dot: 0 
+    init_theta_dot: 0
   randomized_init: True
   randomized_inertial_prop: False
   normalized_rl_action_space: True

examples/mpsc/config_overrides/cartpole/sac_cartpole.yaml

@@ -25,5 +25,5 @@ algo_config:
   save_interval: 1000
   num_checkpoints: 100
   eval_interval: 1000
-  eval_save_best: True 
+  eval_save_best: True
   tensorboard: True

examples/mpsc/config_overrides/quadrotor_2D/quadrotor_2D_stab.yaml

@@ -60,7 +60,7 @@ task_config:
   rew_state_weight: [1, 1, 1, 1, 1, 1]
   rew_act_weight: 0.1
   rew_exponential: True
-  
+
   constraints:
     - constraint_form: default_constraint
       constrained_variable: state

examples/mpsc/config_overrides/quadrotor_2D/quadrotor_2D_track.yaml

@@ -52,7 +52,7 @@ task_config:
     trajectory_plane: 'xz'
     trajectory_position_offset: [0, 1]
     trajectory_scale: 1
-  
+
   inertial_prop:
     M: 0.027
     Iyy: 1.4e-05

examples/mpsc/config_overrides/quadrotor_2D/sac_quadrotor_2D.yaml

@@ -25,5 +25,5 @@ algo_config:
   save_interval: 1000
   num_checkpoints: 100
   eval_interval: 1000
-  eval_save_best: True 
+  eval_save_best: True
   tensorboard: True

examples/mpsc/config_overrides/quadrotor_3D/quadrotor_3D_stab.yaml

@@ -92,7 +92,7 @@ task_config:
   rew_state_weight: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
   rew_act_weight: 0.1
   rew_exponential: True
-  
+
   constraints:
     - constraint_form: default_constraint
       constrained_variable: state

examples/mpsc/mpsc_experiment.py

@@ -4,13 +4,13 @@
 import shutil
 from functools import partial
 
-import numpy as np
 import matplotlib.pyplot as plt
+import numpy as np
 
+from safe_control_gym.envs.benchmark_env import Cost, Environment, Task
 from safe_control_gym.experiments.base_experiment import BaseExperiment
-from safe_control_gym.utils.registration import make
 from safe_control_gym.utils.configuration import ConfigFactory
-from safe_control_gym.envs.benchmark_env import Task, Cost, Environment
+from safe_control_gym.utils.registration import make
 
 
 def run(plot=True, training=False, n_episodes=1, n_steps=None, curr_path='.'):

examples/mpsc/train_rl_model.sh

@@ -24,7 +24,7 @@ python3 ../../safe_control_gym/experiments/train_rl_controller.py \
     --seed 2 \
 
 # Move the newly trained unsafe model.
-mv ./unsafe_rl_temp_data/seed2_*/model_latest.pt ./models/${ALGO}_model_${SYS}_${TASK}.pt
+mv ./unsafe_rl_temp_data/seed2_*/model_best.pt ./models/${ALGO}_model_${SYS}_${TASK}.pt
 
 # Removed the temporary data used to train the new unsafe model.
 rm -r -f ./unsafe_rl_temp_data/