diff --git a/python/setup.py b/python/setup.py
index 7f56d6074..25387f2d8 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -1,6 +1,7 @@
 import glob
 import os
 import sys
+import site
 
 import pybind11
 
@@ -43,6 +44,7 @@ def _maybe_add_library_root(lib_name):
 
 cflags = ["-std=c++17", "-fvisibility=hidden"]
 ldflags = []
+rpaths = []
 package_data = {}
 if sys.platform == "darwin":
     # std::visit requires macOS 10.14
@@ -51,6 +53,12 @@ def _maybe_add_library_root(lib_name):
 elif sys.platform == "win32":
     cflags = ["/std:c++17", "/d2FH4-"]
     package_data["ctranslate2"] = ["*.dll"]
+else:
+    rpaths=[
+        '$ORIGIN/../nvidia/cublas/lib',
+        '$ORIGIN/../nvidia/cuda_runtime/lib',
+        '$ORIGIN/../nvidia/cudnn/lib',
+    ]
 
 ctranslate2_module = Extension(
     "ctranslate2._ext",
@@ -60,6 +68,7 @@ def _maybe_add_library_root(lib_name):
     include_dirs=include_dirs,
     library_dirs=library_dirs,
     libraries=["ctranslate2"],
+    runtime_library_dirs=rpaths
 )
 
 ParallelCompile("CMAKE_BUILD_PARALLEL_LEVEL").install()