diff --git a/setup.py b/setup.py
index b1834fde58..33d5b20810 100755
--- a/setup.py
+++ b/setup.py
@@ -700,7 +700,7 @@ def _check_ld_config(lib: str) -> str:
         return retval
 
     for path in os.environ["LD_LIBRARY_PATH"].split(":"):
-        if not path:
+        if not path or not os.path.exists(path):
             continue
 
         retval = next((fname.strip() for fname in reversed(os.listdir(path))