diff --git a/python/featomic/MANIFEST.in b/python/featomic/MANIFEST.in
index a185f08ab..da1a568ee 100644
--- a/python/featomic/MANIFEST.in
+++ b/python/featomic/MANIFEST.in
@@ -1,5 +1,6 @@
 include featomic-cxx-*.tar.gz
 include git_version_info
+include build-backend/backend.py
 
 include pyproject.toml
 include AUTHORS
diff --git a/python/featomic/build-backend/backend.py b/python/featomic/build-backend/backend.py
new file mode 100644
index 000000000..754daab94
--- /dev/null
+++ b/python/featomic/build-backend/backend.py
@@ -0,0 +1,15 @@
+from setuptools import build_meta
+
+
+get_requires_for_build_sdist = build_meta.get_requires_for_build_sdist
+prepare_metadata_for_build_wheel = build_meta.prepare_metadata_for_build_wheel
+build_wheel = build_meta.build_wheel
+build_sdist = build_meta.build_sdist
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    defaults = build_meta.get_requires_for_build_wheel(config_settings)
+    return defaults + [
+        "cmake",
+        "metatensor-core >=0.1.11,<0.2.0",
+    ]
diff --git a/python/featomic/pyproject.toml b/python/featomic/pyproject.toml
index 55787aaba..afc6c71b5 100644
--- a/python/featomic/pyproject.toml
+++ b/python/featomic/pyproject.toml
@@ -44,9 +44,12 @@ requires = [
     "setuptools",
     "wheel",
     "packaging",
-    "cmake",
 ]
-build-backend = "setuptools.build_meta"
+
+# use a custom build backend to add a dependency on metatensor/cmake only when
+# building the wheels
+build-backend = "backend"
+backend-path = ["build-backend"]
 
 [tool.setuptools]
 zip-safe = true
diff --git a/python/featomic/setup.py b/python/featomic/setup.py
index 400562cb1..642017587 100644
--- a/python/featomic/setup.py
+++ b/python/featomic/setup.py
@@ -1,6 +1,5 @@
 import glob
 import os
-import shutil
 import subprocess
 import sys
 
@@ -45,6 +44,8 @@ class cmake_ext(build_ext):
 
     def run(self):
         """Run cmake build and install the resulting library."""
+        import metatensor
+
         source_dir = FEATOMIC_SRC
         build_dir = os.path.join(ROOT, "build", "cmake-build")
         install_dir = os.path.join(os.path.realpath(self.build_lib), "featomic")
@@ -58,7 +59,7 @@ def run(self):
             f"-DCMAKE_INSTALL_PREFIX={install_dir}",
             "-DCMAKE_INSTALL_LIBDIR=lib",
             f"-DCMAKE_BUILD_TYPE={FEATOMIC_BUILD_TYPE}",
-            "-DFEATOMIC_FETCH_METATENSOR=ON",
+            f"-DCMAKE_PREFIX_PATH={metatensor.utils.cmake_prefix_path}",
             "-DFEATOMIC_INSTALL_BOTH_STATIC_SHARED=OFF",
             "-DBUILD_SHARED_LIBS=ON",
             "-DEXTRA_RUST_FLAGS=-Cstrip=symbols",
@@ -113,19 +114,6 @@ def run(self):
             check=True,
         )
 
-        # do not include metatensor libraries/headers/cmake config within
-        # featomic wheel
-        for file in glob.glob(os.path.join(install_dir, "lib", "libmetatensor.*")):
-            os.unlink(file)
-
-        for file in glob.glob(os.path.join(install_dir, "bin", "metatensor.dll")):
-            os.unlink(file)
-
-        shutil.rmtree(os.path.join(install_dir, "lib", "cmake", "metatensor"))
-
-        for file in glob.glob(os.path.join(install_dir, "include", "metatensor*")):
-            os.unlink(file)
-
 
 class bdist_egg_disabled(bdist_egg):
     """Disabled version of bdist_egg
diff --git a/python/featomic_torch/build-backend/backend.py b/python/featomic_torch/build-backend/backend.py
index ac9375fde..d39eb2526 100644
--- a/python/featomic_torch/build-backend/backend.py
+++ b/python/featomic_torch/build-backend/backend.py
@@ -34,6 +34,7 @@
 def get_requires_for_build_wheel(config_settings=None):
     defaults = build_meta.get_requires_for_build_wheel(config_settings)
     return defaults + [
+        "cmake",
         "torch >= 1.12",
         "metatensor-torch >=0.6.0,<0.7.0",
         FEATOMIC_DEP,
diff --git a/python/featomic_torch/pyproject.toml b/python/featomic_torch/pyproject.toml
index fa92590ab..bec2e4c15 100644
--- a/python/featomic_torch/pyproject.toml
+++ b/python/featomic_torch/pyproject.toml
@@ -37,7 +37,6 @@ requires = [
     "setuptools",
     "wheel",
     "packaging",
-    "cmake",
 ]
 
 # use a custom build backend to add a dependency on the right version of featomic