diff --git a/pyproject.toml b/pyproject.toml
index 779add59e..aa88fcbbc 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -56,10 +56,10 @@ test = [
     "beartype<0.16.0",
     "responses",
     "llama-cpp-python",
-    "mlx-lm",
+    "mlx-lm; platform_machine == 'arm64' and sys_platform == 'darwin'",
     "huggingface_hub",
     "openai>=1.0.0",
-    "vllm",
+    "vllm; sys_platform != 'darwin'",
     "torch",
     "transformers",
 ]