diff --git a/.azure/ci-testig-parameterized.yml b/.azure/ci-testig-parameterized.yml
index ee6c3c40..0116e5a0 100644
--- a/.azure/ci-testig-parameterized.yml
+++ b/.azure/ci-testig-parameterized.yml
@@ -22,4 +22,5 @@ jobs:
     - "Lightning-AI/flash_pl-develop.yaml"
     - "Lightning-AI/flash_pl-release.yaml"
     - "microsoft/deepspeed-release.yaml"
+    - "microsoft/nni-release.yaml"
     - "neptune-ai/lightning_integration.yaml"
diff --git a/actions/assistant.py b/actions/assistant.py
index 8d1235ba..d972ccfb 100644
--- a/actions/assistant.py
+++ b/actions/assistant.py
@@ -187,18 +187,23 @@ def _install_repo(repo: Dict[str, str], remove_dir: bool = True) -> List[str]:
             assert isinstance(repo["checkout"], str)
             cmds.append(f"git checkout {repo['checkout']}")
 
+        cmds += AssistantCLI.before_commands(repo, stage="install", as_append=True)
+
         if "requirements_file" in repo:
             reqs = repo["requirements_file"]
             reqs = [reqs] if isinstance(reqs, str) else reqs
             args = [f"-r {req}" for req in reqs] + ["--quiet", "--upgrade"]
             cmds.append("pip install " + " ".join(args))
 
-        pip_install = "."
-        if "install_extras" in repo:
-            pip_install += f"[{AssistantCLI._extras(repo['install_extras'])}]"
+        if "install_command" in repo:
+            cmds.append(repo["install_command"])
+        else:
+            pip_install = "."
+            if "install_extras" in repo:
+                pip_install += f"[{AssistantCLI._extras(repo['install_extras'])}]"
 
-        flags = AssistantCLI._get_flags(repo)
-        cmds.append("pip install " + " ".join([pip_install] + flags))
+            flags = AssistantCLI._get_flags(repo)
+            cmds.append("pip install " + " ".join([pip_install] + flags))
         cmds.append("pip list")
         cmds.append("cd ..")
 
@@ -226,10 +231,10 @@ def dict_env(config_file: str = "config.yaml") -> str:
 
     @staticmethod
     def before_commands(
-        config_file: str = "config.yaml", stage: str = "install", as_append: bool = False
+        config_file: Union[str, dict] = "config.yaml", stage: str = "install", as_append: bool = False
     ) -> Union[str, List[str]]:
         """Parse commands for eventual custom execution before install or before testing."""
-        config = AssistantCLI._load_config(config_file)
+        config = AssistantCLI._load_config(config_file) if isinstance(config_file, str) else config_file
         cmds = config.get(f"before_{stage}", [])
         if not as_append:
             cmds = os.linesep.join(list(AssistantCLI._BASH_SCRIPT) + cmds)
@@ -277,7 +282,8 @@ def prepare_env(config_file: str = "config.yaml", path_root: str = _PATH_ROOT) -
                 is_file = os.path.splitext(test)[-1] != ""
                 copy_flag = "-r" if not is_file else ""
                 script.append(f'cp {copy_flag} "{repo_test}" "{os.path.join(AssistantCLI._FOLDER_TESTS, test)}"')
-        script.append(f'rm -rf "{repo_name}"')
+        if repo.get("remove_dir", True):
+            script.append(f'rm -rf "{repo_name}"')
 
         reqs = config.get("dependencies", [])
         for req in reqs:
diff --git a/configs/microsoft/nni-release.yaml b/configs/microsoft/nni-release.yaml
new file mode 100644
index 00000000..62ab99bd
--- /dev/null
+++ b/configs/microsoft/nni-release.yaml
@@ -0,0 +1,37 @@
+target_repository:
+  HTTPS: https://github.com/microsoft/nni.git
+  # checkout a particular branch or a tag
+  checkout: master
+  before_install:
+    - if [ -x "$(command -v nvidia-smi)" ]; then mv dependencies/recommended_gpu.txt dependencies/recommended.txt; fi
+  # NNI needs custom install. `python setup.py install` or `pip install .` won't work.
+  requirements_file:
+    - dependencies/develop.txt
+    - dependencies/required.txt
+    - dependencies/required_extra.txt
+    - dependencies/recommended.txt
+  # install_command: python setup.py develop
+  # development build. Cannot remove dir
+  # remove_dir: false
+  # copy some tests from the target repository
+  copy_tests:
+    # only retiarii uses pytorch-lightning for now
+    # might add more in future
+    - test/ut/retiarii
+    - test/ut/__init__.py
+
+before_install:
+  - pip install numpy
+
+dependencies:
+  - name: pytorch-lightning
+    HTTPS: https://github.com/PyTorchLightning/pytorch-lightning.git
+    checkout: release/1.5.x
+    install_extras: loggers
+
+testing:
+  dirs:
+    - test/ut
+
+runtimes:
+  - {os: "ubuntu-latest", python-version: "3.8"}