amend

Signed-off-by: Matteo Bettini <[email protected]>

.github/unittest/install_dependencies.sh

@@ -0,0 +1,14 @@
+
+
+python -m pip install --upgrade pip
+python -m pip install flake8 pytest pytest-cov
+if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+python -m pip install --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cpu --force-reinstall
+
+cd ..
+python -m pip install git+https://github.com/pytorch-labs/tensordict.git
+git clone https://github.com/pytorch/rl.git
+cd rl
+python setup.py develop
+cd ../BenchMARL
+pip install -e .

.github/unittest/install_vmas.sh

@@ -0,0 +1,2 @@
+
+python -m pip install vmas

.github/workflows/python-app.yml → .github/workflows/lint.yml

@@ -3,11 +3,11 @@
 # https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 
-name: pytest
+name: lint
 
 on:
   push:
-    branches: [ $default-branch , "main" , "dev" ]
+    branches: [ $default-branch , "main" ]
   pull_request:
     branches: [ $default-branch , "main" ]
 
@@ -20,39 +20,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.10"]
 
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install flake8 pytest
-        python -m pip install vmas
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        python -m pip install --pre torch --extra-index-url https://download.pytorch.org/whl/nightly/cpu --force-reinstall
-        
-        cd ..
-        python -m pip install git+https://github.com/pytorch-labs/tensordict.git
-        git clone https://github.com/pytorch/rl.git
-        cd rl
-        python setup.py develop
-        cd ../BenchMARL
-        
-        pip install -e .
+
     - name: Lint with flake8
       run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8
         # stop the build if there are Python syntax errors or undefined names
         flake8 benchmarl/ --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 benchmarl/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore=E203,W503
-    - name: Test with pytest
-      run: |
-        pip install pytest 
-        pip install pytest-cov
-        pip install tqdm
-        pytest test/ --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html

.github/workflows/vmas_tests.yml

@@ -0,0 +1,40 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see:
+# https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+
+name: vmas_tests
+
+on:
+  push:
+    branches: [ $default-branch , "main" ]
+  pull_request:
+    branches: [ $default-branch , "main" ]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+       bash .github/unittest/install_dependencies.sh
+    - name: Install vmas
+      run: |
+       bash .github/unittest/install_vmas.sh
+
+    - name: Test with pytest
+      run: |
+        pytest test/test_vmas.py --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=xml --cov-report=html

test/conftest.py

@@ -1,6 +1,33 @@
+import os
+import shutil
+from pathlib import Path
+
 import pytest
 
 from benchmarl.experiment import ExperimentConfig
+from benchmarl.models import MlpConfig
+from benchmarl.models.common import ModelConfig, SequenceModelConfig
+from torch import nn
+
+
+def pytest_sessionstart(session):
+    """
+    Called after the Session object has been created and
+    before performing collection and entering the run test loop.
+    """
+    folder_name = Path(os.getcwd())
+    folder_name = folder_name / "tmp"
+    folder_name.mkdir(parents=False, exist_ok=True)
+    os.chdir(folder_name)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    """
+    Called after whole test run finished, right before
+    returning the exit status to the system.
+    """
+    folder_name = Path(os.getcwd()) / "tmp"
+    shutil.rmtree(folder_name)
 
 
 @pytest.fixture
@@ -13,8 +40,20 @@ def experiment_config() -> ExperimentConfig:
     experiment_config.on_policy_minibatch_size = 10
     experiment_config.off_policy_memory_size = 200
     experiment_config.off_policy_train_batch_size = 100
-    experiment_config.evaluation = False
-    experiment_config.loggers = []
-    experiment_config.create_json = False
-    experiment_config.checkpoint_interval = 0
+    experiment_config.evaluation = True
+    experiment_config.evaluation_episodes = 2
+    experiment_config.loggers = ["csv"]
+    experiment_config.create_json = True
+    experiment_config.checkpoint_interval = 1
     return experiment_config
+
+
+@pytest.fixture
+def mlp_sequence_config() -> ModelConfig:
+    return SequenceModelConfig(
+        model_configs=[
+            MlpConfig(num_cells=[8], activation_class=nn.Tanh, layer_class=nn.Linear),
+            MlpConfig(num_cells=[4], activation_class=nn.Tanh, layer_class=nn.Linear),
+        ],
+        intermediate_sizes=[5],
+    )

test/test_smac.py

@@ -0,0 +1,51 @@
+import importlib
+
+import pytest
+
+from benchmarl.algorithms import (
+    algorithm_config_registry,
+    MappoConfig,
+    MasacConfig,
+    QmixConfig,
+)
+from benchmarl.algorithms.common import AlgorithmConfig
+from benchmarl.environments import Smacv2Task
+from benchmarl.experiment import Experiment
+
+
+_has_smacv2 = importlib.util.find_spec("smacv2") is not None
+
+
+@pytest.mark.skipif(not _has_smacv2, reason="SMACv2 not found")
+class TestSmacv2:
+    @pytest.mark.parametrize("algo_config", algorithm_config_registry.values())
+    @pytest.mark.parametrize("task", list(Smacv2Task))
+    def test_all_algos(
+        self, algo_config: AlgorithmConfig, task, experiment_config, mlp_sequence_config
+    ):
+        if algo_config.supports_discrete_actions():
+            task = task.get_from_yaml()
+
+            experiment = Experiment(
+                algorithm_config=algo_config.get_from_yaml(),
+                model_config=mlp_sequence_config,
+                seed=0,
+                config=experiment_config,
+                task=task,
+            )
+            experiment.run()
+
+    @pytest.mark.parametrize("algo_config", [QmixConfig, MappoConfig, MasacConfig])
+    @pytest.mark.parametrize("task", list(Smacv2Task))
+    def test_all_tasks(
+        self, algo_config: AlgorithmConfig, task, experiment_config, mlp_sequence_config
+    ):
+        task = task.get_from_yaml()
+        experiment = Experiment(
+            algorithm_config=algo_config.get_from_yaml(),
+            model_config=mlp_sequence_config,
+            seed=0,
+            config=experiment_config,
+            task=task,
+        )
+        experiment.run()

test/test_vmas.py

@@ -0,0 +1,37 @@
+import importlib
+
+import pytest
+
+from benchmarl.algorithms import algorithm_config_registry
+from benchmarl.algorithms.common import AlgorithmConfig
+from benchmarl.environments import Task, VmasTask
+from benchmarl.experiment import Experiment
+
+_has_vmas = importlib.util.find_spec("vmas") is not None
+
+
+@pytest.mark.skipif(not _has_vmas, reason="VMAS not found")
+class TestVmas:
+    @pytest.mark.parametrize("algo_config", algorithm_config_registry.values())
+    @pytest.mark.parametrize("continuous", [True])
+    @pytest.mark.parametrize("task", list(VmasTask))
+    def test_all_algos_all_tasks(
+        self,
+        algo_config: AlgorithmConfig,
+        task: Task,
+        continuous,
+        experiment_config,
+        mlp_sequence_config,
+    ):
+        task = task.get_from_yaml()
+
+        experiment_config.prefer_continuous_actions = continuous
+
+        experiment = Experiment(
+            algorithm_config=algo_config.get_from_yaml(),
+            model_config=mlp_sequence_config,
+            seed=0,
+            config=experiment_config,
+            task=task,
+        )
+        experiment.run()