ENH: set __path__ for module loaded in editable installation

Set the __path__ module attribute to a placeholder path.  Because how
editable installs are implemented, this cannot correspond to the
filesystem path for the package.  And add a sys.path_hook that
recognizes this paths and returns a path loader that implements
iter_modules(). This allows pkgutil.iter_packages() to work properly.

Fixes mesonbuild#557.

mesonpy/_editable.py

@@ -10,6 +10,7 @@
 import importlib.abc
 import importlib.machinery
 import importlib.util
+import inspect
 import json
 import os
 import pathlib
@@ -182,7 +183,7 @@ def build_module_spec(cls: type, name: str, path: str, tree: Optional[Node]) ->
     spec = importlib.machinery.ModuleSpec(name, loader, origin=path)
     spec.has_location = True
     if loader.is_package(name):
-        spec.submodule_search_locations = []
+        spec.submodule_search_locations = [os.path.join(__file__, name)]
     return spec
 
 
@@ -317,6 +318,41 @@ def _rebuild(self) -> Node:
             install_plan = json.load(f)
         return collect(install_plan)
 
+    def _path_hook(self, path: str) -> MesonpyPathFinder:
+        if os.altsep:
+            path.replace(os.altsep, os.sep)
+        path, _, key = path.rpartition(os.sep)
+        if path == __file__:
+            tree = self._rebuild()
+            node = tree.get(tuple(key.split('.')))
+            if isinstance(node, Node):
+                return MesonpyPathFinder(node)
+        raise ImportError
+
+
+class MesonpyPathFinder(importlib.abc.PathEntryFinder):
+    def __init__(self, tree: Node):
+        self._tree = tree
+
+    def iter_modules(self, prefix: str) -> Iterator[Tuple[str, bool]]:
+        yielded = set()
+        for name, node in self._tree.items():
+            ispkg = False
+            modname = inspect.getmodulename(name)
+            if modname == '__init__' or modname in yielded:
+                continue
+            if isinstance(node, Node):
+                modname = name
+                for _, suffix in LOADERS:
+                    src = node.get('__init__' + suffix)
+                    if isinstance(src, str):
+                        ispkg = True
+            if modname and '.' not in modname:
+                yielded.add(modname)
+                yield prefix + modname, ispkg
+
 
 def install(names: Set[str], path: str, cmd: List[str], verbose: bool) -> None:
-    sys.meta_path.insert(0, MesonpyMetaFinder(names, path, cmd, verbose))
+    finder = MesonpyMetaFinder(names, path, cmd, verbose)
+    sys.meta_path.insert(0, finder)
+    sys.path_hooks.insert(0, finder._path_hook)

tests/packages/complex/complex/more/baz.pyx

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2023 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+def answer():
+    return 42

tests/packages/complex/complex/more/meson.build

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2024 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+py.extension_module(
+    'baz',
+    'baz.pyx',
+    install: true,
+    subdir: 'complex/more',
+)

tests/packages/complex/foo.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2024 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+def foo():
+    return True

tests/packages/complex/meson.build

@@ -15,6 +15,13 @@ endif
 
 py = import('python').find_installation()
 
+py.install_sources('move.py', subdir: 'complex/more', pure: false)
+
+install_data('foo.py', rename: 'bar.py', install_dir: py.get_install_dir(pure: false) / 'complex')
+
 install_subdir('complex', install_dir: py.get_install_dir(pure: false))
 
 py.extension_module('test', 'test.pyx', install: true, subdir: 'complex')
+py.extension_module('baz', 'complex/more/baz.pyx', install: true, subdir: 'complex/more')
+
+subdir('complex/more')

tests/packages/complex/move.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: 2024 The meson-python developers
+#
+# SPDX-License-Identifier: MIT
+
+def test():
+    return True

tests/test_editable.py

@@ -4,6 +4,7 @@
 
 import os
 import pathlib
+import pkgutil
 import sys
 
 import pytest
@@ -189,3 +190,31 @@ def test_editble_reentrant(venv, editable_imports_itself_during_build):
         assert venv.python('-c', 'import plat; print(plat.data())').strip() == 'DEF'
     finally:
         path.write_text(code)
+
+
+def test_editable_pkgutils_walk_packages(package_complex, tmp_path):
+    # build a package in a temporary directory
+    mesonpy.Project(package_complex, tmp_path)
+
+    finder = _editable.MesonpyMetaFinder({'complex'}, os.fspath(tmp_path), ['ninja'])
+
+    try:
+        # install editable hooks
+        sys.meta_path.insert(0, finder)
+        sys.path_hooks.insert(0, finder._path_hook)
+
+        import complex
+        packages = {m.name for m in pkgutil.walk_packages(complex.__path__, complex.__name__ + '.')}
+        assert packages == {
+            'complex.bar',
+            'complex.more',
+            'complex.more.baz',
+            'complex.more.move',
+            'complex.namespace',
+            'complex.test',
+        }
+
+    finally:
+        # remove hooks
+        del sys.meta_path[0]
+        del sys.path_hooks[0]