diff --git a/docs/tools.rst b/docs/tools.rst index b7994e01..883d4456 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -15,7 +15,20 @@ your python package. This will scan all of your defined includes directories (including those of downloaded artifacts) and output something you can paste into the ``generate`` -key of ``pyproject.toml``. +key of ``pyproject.toml``. By default it will only show files that are not +present in ``pyproject.toml`` -- to show all files use the ``--all`` argument. + +Often there are files that you don't want to wrap. You can add them to the +``pyproject.toml`` file and they will be ignored. The list accepts glob patterns +supported by the fnmatch module. + +.. code-block:: toml + + [tool.robotpy-build] + scan_headers_ignore = [ + "ignored_header.h", + "ignore_dir/*", + ] .. _create_gen: @@ -40,3 +53,15 @@ python package. .. code-block:: sh $ python -m robotpy_build create-imports rpydemo rpydemo._rpydemo + +Use the ``--write`` argument to write the file. + +To write a list of ``__init__.py`` files, you can specify them in the ``pyproject.toml`` +file like so: + +.. code-block:: toml + + [tool.robotpy-build] + update_init = ["rpydemo rpydemo._rpydemo"] + +To actually update the files, run ``python setup.py update_init``. \ No newline at end of file diff --git a/robotpy_build/autowrap/generator_data.py b/robotpy_build/autowrap/generator_data.py index fcf051cf..be01bcf2 100644 --- a/robotpy_build/autowrap/generator_data.py +++ b/robotpy_build/autowrap/generator_data.py @@ -133,7 +133,7 @@ def get_function_data( data = self._default_fn_data report_data.deferred_signatures.append((fn, is_private)) elif not data.overloads: - report_data.deferred_signatures.append((fn, True)) + report_data.deferred_signatures.append((fn, is_private)) else: # When there is overload data present, we have to actually compute # the signature of every function diff --git a/robotpy_build/command/_built_env.py b/robotpy_build/command/_built_env.py new file mode 100644 index 00000000..91e8608d --- /dev/null +++ b/robotpy_build/command/_built_env.py @@ -0,0 +1,79 @@ +import importlib.util +import os +from os.path import abspath, exists, dirname, join + +from setuptools import Command + +from .util import get_install_root + + +class _BuiltEnv(Command): + + user_options = [("build-lib=", "d", 'directory to "build" (copy) to')] + + def initialize_options(self): + self.build_lib = None + + def finalize_options(self): + self.set_undefined_options("build", ("build_lib", "build_lib")) + + def setup_built_env(self): + + # Gather information for n + data = {"mapping": {}} + + # OSX-specific: need to set DYLD_LIBRARY_PATH otherwise modules don't + # work. Luckily, that information was computed when building the + # extensions... + env = os.environ.copy() + dyld_path = set() + + # Requires information from build_ext to work + build_ext = self.get_finalized_command("build_ext") + if build_ext.inplace: + data["out"] = get_install_root(self) + else: + data["out"] = self.build_lib + + # Ensure that the associated packages can always be found locally + for wrapper in build_ext.wrappers: + pkgdir = wrapper.package_name.split(".") + init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py")) + if exists(init_py): + data["mapping"][wrapper.package_name] = init_py + + # Ensure that the built extension can always be found + build_ext.resolve_libs() + for ext in build_ext.extensions: + fname = build_ext.get_ext_filename(ext.name) + data["mapping"][ext.name] = abspath(join(self.build_lib, fname)) + + rpybuild_libs = getattr(ext, "rpybuild_libs", None) + if rpybuild_libs: + for pth, _ in rpybuild_libs.values(): + dyld_path.add(dirname(pth)) + + # OSX-specific + if dyld_path: + dyld_path = ":".join(dyld_path) + if "DYLD_LIBRARY_PATH" in env: + dyld_path += ":" + env["DYLD_LIBRARY_PATH"] + env["DYLD_LIBRARY_PATH"] = dyld_path + + return data, env + + +class _PackageFinder: + """ + Custom loader to allow loading built modules from their location + in the build directory (as opposed to their install location) + """ + + # Set this to mapping returned from _BuiltEnv.setup_built_env + mapping = {} + + @classmethod + def find_spec(cls, fullname, path, target=None): + m = cls.mapping.get(fullname) + if m: + return importlib.util.spec_from_file_location(fullname, m) diff --git a/robotpy_build/command/build_ext.py b/robotpy_build/command/build_ext.py index 33e4a08a..5b7ff6c0 100644 --- a/robotpy_build/command/build_ext.py +++ b/robotpy_build/command/build_ext.py @@ -158,6 +158,27 @@ def _spawn(cmd): # Used in build_pyi ext.rpybuild_libs = libs + def resolve_libs(self): + # used in _built_env + platform = get_platform() + if platform.os == "osx": + for wrapper in self.wrappers: + wrapper.finalize_extension() + + from ..relink_libs import resolve_libs + + install_root = get_install_root(self) + + for ext in self.extensions: + libs = resolve_libs( + install_root, + ext.rpybuild_wrapper, + self.rpybuild_pkgcfg, + ) + + # Used in build_pyi + ext.rpybuild_libs = libs + def run(self): # files need to be generated before building can occur self.run_command("build_gen") diff --git a/robotpy_build/command/build_pyi.py b/robotpy_build/command/build_pyi.py index f75fbcba..3eaaf3a1 100644 --- a/robotpy_build/command/build_pyi.py +++ b/robotpy_build/command/build_pyi.py @@ -1,35 +1,29 @@ -import importlib.util import json import os -from os.path import abspath, exists, dirname, join +from os.path import exists, dirname, join import subprocess import sys import pybind11_stubgen -from setuptools import Command -from distutils.errors import DistutilsError -from .util import get_install_root +try: + from setuptools.errors import BaseError +except ImportError: + from distutils.errors import DistutilsError as BaseError +from ._built_env import _BuiltEnv, _PackageFinder -class GeneratePyiError(DistutilsError): + +class GeneratePyiError(BaseError): pass -class BuildPyi(Command): +class BuildPyi(_BuiltEnv): base_package: str command_name = "build_pyi" description = "Generates pyi files from built extensions" - user_options = [("build-lib=", "d", 'directory to "build" (copy) to')] - - def initialize_options(self): - self.build_lib = None - - def finalize_options(self): - self.set_undefined_options("build", ("build_lib", "build_lib")) - def run(self): # cannot build pyi files when cross-compiling if ( @@ -40,50 +34,18 @@ def run(self): return # Gather information for needed stubs - data = {"mapping": {}, "stubs": []} - - # OSX-specific: need to set DYLD_LIBRARY_PATH otherwise modules don't - # work. Luckily, that information was computed when building the - # extensions... - env = os.environ.copy() - dyld_path = set() - - # Requires information from build_ext to work - build_ext = self.distribution.get_command_obj("build_ext") - if build_ext.inplace: - data["out"] = get_install_root(self) - else: - data["out"] = self.build_lib - - # Ensure that the associated packages can always be found locally - for wrapper in build_ext.wrappers: - pkgdir = wrapper.package_name.split(".") - init_py = abspath(join(self.build_lib, *pkgdir, "__init__.py")) - if exists(init_py): - data["mapping"][wrapper.package_name] = init_py + data, env = self.setup_built_env() + data["stubs"] = [] # Ensure that the built extension can always be found + build_ext = self.get_finalized_command("build_ext") for ext in build_ext.extensions: - fname = build_ext.get_ext_filename(ext.name) - data["mapping"][ext.name] = abspath(join(self.build_lib, fname)) data["stubs"].append(ext.name) - rpybuild_libs = getattr(ext, "rpybuild_libs", None) - if rpybuild_libs: - for pth, _ in rpybuild_libs.values(): - dyld_path.add(dirname(pth)) - # Don't do anything if nothing is needed if not data["stubs"]: return - # OSX-specific - if dyld_path: - dyld_path = ":".join(dyld_path) - if "DYLD_LIBRARY_PATH" in env: - dyld_path += ":" + env["DYLD_LIBRARY_PATH"] - env["DYLD_LIBRARY_PATH"] = dyld_path - data_json = json.dumps(data) # Execute in a subprocess in case it crashes @@ -101,45 +63,6 @@ def run(self): pass -class _PackageFinder: - """ - Custom loader to allow loading built modules from their location - in the build directory (as opposed to their install location) - """ - - mapping = {} - - @classmethod - def find_spec(cls, fullname, path, target=None): - m = cls.mapping.get(fullname) - if m: - return importlib.util.spec_from_file_location(fullname, m) - - -def generate_pyi(module_name: str, pyi_filename: str): - print("generating", pyi_filename) - - pybind11_stubgen.FunctionSignature.n_invalid_signatures = 0 - module = pybind11_stubgen.ModuleStubsGenerator(module_name) - module.parse() - if pybind11_stubgen.FunctionSignature.n_invalid_signatures > 0: - print("FAILED to generate pyi for", module_name, file=sys.stderr) - return False - - module.write_setup_py = False - with open(pyi_filename, "w") as fp: - fp.write("#\n# AUTOMATICALLY GENERATED FILE, DO NOT EDIT!\n#\n\n") - fp.write("\n".join(module.to_lines())) - - typed = join(dirname(pyi_filename), "py.typed") - print("generating", typed) - if not exists(typed): - with open(typed, "w") as fp: - pass - - return True - - def main(): cfg = json.load(sys.stdin) diff --git a/robotpy_build/command/update_init.py b/robotpy_build/command/update_init.py new file mode 100644 index 00000000..6c58d2b5 --- /dev/null +++ b/robotpy_build/command/update_init.py @@ -0,0 +1,73 @@ +import json +import os +import subprocess +import sys +import typing + +try: + from setuptools.errors import BaseError +except ImportError: + from distutils.errors import DistutilsError as BaseError + +from ._built_env import _BuiltEnv, _PackageFinder + + +class UpdateInitError(BaseError): + pass + + +class UpdateInit(_BuiltEnv): + update_list: typing.List[str] + + command_name = "update_init" + description = ( + "Updates __init__.py files using settings from tool.robotpy-build.update_init" + ) + + def run(self): + # cannot use when cross-compiling + if ( + "_PYTHON_HOST_PLATFORM" in os.environ + or "PYTHON_CROSSENV" in os.environ + or not self.update_list + ): + return + + data, env = self.setup_built_env() + data["update_list"] = self.update_list + + data_json = json.dumps(data) + + # Execute in a subprocess in case it crashes + args = [sys.executable, "-m", __name__] + try: + subprocess.run(args, input=data_json.encode("utf-8"), env=env, check=True) + except subprocess.CalledProcessError: + raise UpdateInitError( + "Failed to generate .pyi file (see above, or set RPYBUILD_SKIP_PYI=1 to ignore) via %s" + % (args,) + ) from None + + +def main(): + cfg = json.load(sys.stdin) + + # Configure custom loader + _PackageFinder.mapping = cfg["mapping"] + sys.meta_path.insert(0, _PackageFinder) + + from .. import tool + + # Update init + + for to_update in cfg["update_list"]: + + sys.argv = ["", "create-imports", "-w"] + to_update.split(" ", 1) + + retval = tool.main() + if retval != 0: + break + + +if __name__ == "__main__": + main() diff --git a/robotpy_build/config/pyproject_toml.py b/robotpy_build/config/pyproject_toml.py index dff02090..67898589 100644 --- a/robotpy_build/config/pyproject_toml.py +++ b/robotpy_build/config/pyproject_toml.py @@ -416,6 +416,14 @@ class RobotpyBuildConfig(Model): #: Python package to store version information and robotpy-build metadata in base_package: str + #: List of headers for the scan-headers tool to ignore + scan_headers_ignore: List[str] = [] + + #: List of python packages with __init__.py to update when ``python setup.py update_init`` + #: is called -- this is an argument to the ``robotpy-build create-imports`` command, and + #: may contain a space and the second argument to create-imports. + update_init: List[str] = [] + #: #: .. seealso:: :class:`.SupportedPlatform` #: diff --git a/robotpy_build/relink_libs.py b/robotpy_build/relink_libs.py index adcca42b..ddbaa823 100644 --- a/robotpy_build/relink_libs.py +++ b/robotpy_build/relink_libs.py @@ -164,3 +164,14 @@ def relink_extension( } _fix_libs(to_fix, libs) return libs + + +def resolve_libs( + install_root: str, + pkg: PkgCfg, + pkgcfg: PkgCfgProvider, +): + libs: LibsDict = {} + _resolve_dependencies(install_root, pkg, pkgcfg, libs) + _resolve_libs_in_self(pkg, install_root, libs) + return libs diff --git a/robotpy_build/setup.py b/robotpy_build/setup.py index c1bbe022..75912c29 100644 --- a/robotpy_build/setup.py +++ b/robotpy_build/setup.py @@ -23,6 +23,7 @@ def finalize_options(self): from .command.build_ext import BuildExt from .command.build_pyi import BuildPyi from .command.develop import Develop +from .command.update_init import UpdateInit try: from .command.editable_wheel import EditableWheel @@ -155,6 +156,7 @@ def _xform(v: str): "build_ext": BuildExt, "build_pyi": BuildPyi, "develop": Develop, + "update_init": UpdateInit, } if EditableWheel: self.setup_kwargs["cmdclass"]["editable_wheel"] = EditableWheel @@ -165,6 +167,7 @@ def _xform(v: str): cls.static_libs = self.static_libs cls.rpybuild_pkgcfg = self.pkgcfg BuildPyi.base_package = self.base_package + UpdateInit.update_list = self.project.update_init # We already know some of our packages, so collect those in addition # to using find_packages() diff --git a/robotpy_build/static_libs.py b/robotpy_build/static_libs.py index 6ea3d148..0f0680bc 100644 --- a/robotpy_build/static_libs.py +++ b/robotpy_build/static_libs.py @@ -33,6 +33,9 @@ def set_root(self, root: os.PathLike) -> None: self.incdir = join(self.root, self.name, "include") def get_include_dirs(self) -> Optional[List[str]]: + if self.incdir is None: + return + includes = [self.incdir] if self.cfg.download: for dl in self.cfg.download: @@ -41,7 +44,8 @@ def get_include_dirs(self) -> Optional[List[str]]: return includes def get_library_dirs(self) -> Optional[List[str]]: - return [self.libdir] + if self.libdir: + return [self.libdir] def get_library_dirs_rel(self) -> Optional[List[str]]: pass @@ -60,7 +64,8 @@ def get_extra_objects(self) -> Optional[List[str]]: if self.platform.os == "windows": return - return [join(self.libdir, lib) for lib in self._get_libnames()] + if self.libdir: + return [join(self.libdir, lib) for lib in self._get_libnames()] def get_type_casters_cfg(self, casters: Dict[str, Dict[str, Any]]) -> None: pass diff --git a/robotpy_build/tool.py b/robotpy_build/tool.py index ca3d9b5f..77903e8e 100644 --- a/robotpy_build/tool.py +++ b/robotpy_build/tool.py @@ -1,4 +1,5 @@ import argparse +import fnmatch import glob import inspect from itertools import chain @@ -9,6 +10,7 @@ import subprocess import sys import types +import typing import re from urllib.request import Request, urlopen from urllib.error import HTTPError @@ -28,11 +30,6 @@ def get_setup() -> Setup: s = Setup() s.prepare() - - temp_path = join(get_build_temp_path(), "dlstatic") - for static_lib in s.static_libs: - static_lib.set_root(temp_path) - return s @@ -136,14 +133,22 @@ def add_subparser(cls, parent_parser, subparsers): help="Generate a list of headers in TOML form", parents=[parent_parser], ) - parser.add_argument("--only-missing", default=False, action="store_true") + parser.add_argument("--all", default=False, action="store_true") return parser def run(self, args): s = get_setup() + to_ignore = s.project.scan_headers_ignore + + def _should_ignore(f): + for pat in to_ignore: + if fnmatch.fnmatch(f, pat): + return True + return False + already_present = {} - if args.only_missing: + if not args.all: for i, wrapper in enumerate(s.project.wrappers.values()): files = set() if wrapper.autogen_headers: @@ -160,9 +165,7 @@ def run(self, args): ifiles.add(f) for wrapper in s.wrappers: - print( - f'[tool.robotpy-build.wrappers."{wrapper.package_name}".autogen_headers]' - ) + printed = False # This uses the direct include directories instead of the generation # search path as we only want to output a file once @@ -176,15 +179,22 @@ def run(self, args): glob.glob(join(incdir, "**", "*.h"), recursive=True), glob.glob(join(incdir, "**", "*.hpp"), recursive=True), ) - if "rpygen" not in f + if "rpygen" not in f and not _should_ignore(relpath(f, incdir)) ) ) + files = [f for f in files if f not in wpresent] + if not files: + continue + + if not printed: + print( + f'[tool.robotpy-build.wrappers."{wrapper.package_name}".autogen_headers]' + ) + printed = True + lastdir = None for f in files: - if f in wpresent: - continue - thisdir = f.parent if lastdir is None: if thisdir: @@ -210,6 +220,9 @@ def add_subparser(cls, parent_parser, subparsers): ) parser.add_argument("base", help="Ex: wpiutil") parser.add_argument("compiled", nargs="?", help="Ex: wpiutil._impl.wpiutil") + parser.add_argument( + "--write", "-w", action="store_true", help="Modify existing __init__.py" + ) return parser def _rel(self, base: str, compiled: str) -> str: @@ -220,6 +233,9 @@ def _rel(self, base: str, compiled: str) -> str: return f".{'.'.join(elems)}" def run(self, args): + self.create(args.base, args.compiled, args.write) + + def create(self, base: str, compiled: typing.Optional[str], write: bool): # Runtime Dependency Check try: import black @@ -227,9 +243,8 @@ def run(self, args): print("Error, The following module is required to run this tool: black") exit(1) - compiled = args.compiled if not compiled: - compiled = f"{args.base}._{args.base.split('.')[-1]}" + compiled = f"{base}._{base.split('.')[-1]}" # TODO: could probably generate this from parsed code, but seems hard ctx = {} @@ -238,25 +253,57 @@ def run(self, args): if isinstance(ctx[k], types.ModuleType): del ctx[k] - relimport = self._rel(args.base, compiled) + relimport = self._rel(base, compiled) - stmt_compiled = "" if not args.compiled else f" {args.compiled}" + stmt_compiled = "" if not compiled else f" {compiled}" + begin_stmt = f"# autogenerated by 'robotpy-build create-imports {base}" stmt = inspect.cleandoc( f""" - # autogenerated by 'robotpy-build create-imports {args.base}{stmt_compiled}' + {begin_stmt}{stmt_compiled}' from {relimport} import {','.join(sorted(ctx.keys()))} __all__ = ["{'", "'.join(sorted(ctx.keys()))}"] """ ) - print( - subprocess.check_output( - ["black", "-", "-q"], input=stmt.encode("utf-8") - ).decode("utf-8") - ) + content = subprocess.check_output( + ["black", "-", "-q"], input=stmt.encode("utf-8") + ).decode("utf-8") + + if write: + fctx = {} + exec(f"from {base} import __file__", {}, fctx) + fname = fctx["__file__"] + + with open(fname) as fp: + fcontent = orig_content = fp.read() + + # Find the beginning statement + idx = startidx = fcontent.find(begin_stmt) + if startidx != -1: + for to_find in ("from", "__all__", "[", "]", "\n"): + idx = fcontent.find(to_find, idx) + if idx == -1: + startidx = -1 + break + + if startidx == -1: + # If not present, just append and let the user figure it out + fcontent = fcontent + "\n" + content + else: + fcontent = fcontent[:startidx] + content + fcontent[idx + 1 :] + + if fcontent != orig_content: + with open(fname, "w") as fp: + fp.write(fcontent) + print("MOD", base) + else: + print("OK", base) + + else: + print(content) class PlatformInfo: @@ -506,8 +553,8 @@ def main(): else: retval = 0 - exit(retval) + return retval if __name__ == "__main__": - main() + exit(main()) diff --git a/robotpy_build/wrapper.py b/robotpy_build/wrapper.py index f7efb0e0..33e72552 100644 --- a/robotpy_build/wrapper.py +++ b/robotpy_build/wrapper.py @@ -288,7 +288,9 @@ def all_deps(self): def _all_includes(self, include_rpyb): includes = self.get_include_dirs() for dep in self.all_deps(): - includes.extend(dep.get_include_dirs()) + dep_inc = dep.get_include_dirs() + if dep_inc: + includes.extend(dep_inc) if include_rpyb: includes.extend(self.pkgcfg.get_pkg("robotpy-build").get_include_dirs()) return includes