diff --git a/pyproject.toml b/pyproject.toml
index e5dcd222f1..334099f910 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,4 +10,7 @@ check-return-types = 'False'
 exclude = 'tests/torchtune/models/llama2/scripts/'
 
 [tool.pytest.ini_options]
-addopts = ["--showlocals"] # show local variables in tracebacks
+addopts = ["--showlocals", "--import-mode=importlib"]
+# --showlocals will show local variables in tracebacks
+# --import-model=importlib ensures pytest doesn't append the repo root to sys.path,
+# causing our tests to bypass legitimate import errors
diff --git a/tests/recipes/configs/test_configs.py b/tests/recipes/configs/test_configs.py
deleted file mode 100644
index ab259f8b1b..0000000000
--- a/tests/recipes/configs/test_configs.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
-import os
-
-import pytest
-from omegaconf import OmegaConf
-from recipes.full_finetune import FullFinetuneRecipe
-from recipes.lora_finetune import LoRAFinetuneRecipe
-
-from torchtune.utils.argparse import TuneArgumentParser
-
-ROOT_DIR: str = os.path.join(os.path.abspath(__file__), "../../../configs")
-
-config_to_recipe = {
-    os.path.join(ROOT_DIR, "alpaca_llama2_full_finetune.yaml"): FullFinetuneRecipe,
-    os.path.join(ROOT_DIR, "alpaca_llama2_lora_finetune.yaml"): LoRAFinetuneRecipe,
-}
-
-
-class TestConfigs:
-    """Tests that all configs are well formed.
-    Configs should have the complete set of arguments as specified by the recipe.
-    """
-
-    @pytest.fixture
-    def parser(self):
-        parser = TuneArgumentParser("Test parser")
-        return parser
-
-    # TODO: update this test to run recipes with debug args, disabling for now
-    @pytest.mark.skip(
-        reason="Need to update to use debug args after config system is finalized."
-    )
-    def test_configs(self, parser) -> None:
-        for config_path, recipe in config_to_recipe.items():
-            args, _ = parser.parse_known_args(["--config", config_path])
-            try:
-                cfg = OmegaConf.create(vars(args))
-                recipe(cfg)
-            except ValueError as e:
-                raise AssertionError(
-                    f"Config {config_path} for recipe {recipe.__name__} is not well formed"
-                ) from e