diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3e5750c0..8b0ae567 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,7 +8,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10"] + python-version: ["3.12"] steps: - uses: actions/checkout@v3 diff --git a/README.rst b/README.rst index 12ae0c99..520fd360 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,36 @@ or [PYFLYBY] from base64 import b64decode Out[1]: 'hello' +Auto importer lazy variables +---------------------------- + +It is possible to use the autoimporter to lazily define variables. + +To use, put the following in your IPython startup files +(``~/.ipython/profile_default/startup/autoimp.py``), or in your IPython +configuration file: + +.. code:: python + + + from pyflyby import add_import + + add_import("foo", "foo = 1") + + add_import( + "df, data as dd", + ''' + import pandas as pd + data = [1,2,3] + df = pd.DataFrame(data) + ''') + + +You can add the keyword ``strict=False`` to not fail if not in IPython or of the +pyflyby extensions is not loaded. + + + Quick start: ``py`` command-line multi-tool =========================================== diff --git a/lib/python/pyflyby/__init__.py b/lib/python/pyflyby/__init__.py index da88e994..b31abf4d 100644 --- a/lib/python/pyflyby/__init__.py +++ b/lib/python/pyflyby/__init__.py @@ -6,6 +6,7 @@ from pyflyby._autoimp import (auto_eval, auto_import, find_missing_imports) +from pyflyby._dynimp import add_import from pyflyby._dbg import (add_debug_functions_to_builtins, attach_debugger, debug_on_exception, debug_statement, debugger, diff --git a/lib/python/pyflyby/_autoimp.py b/lib/python/pyflyby/_autoimp.py index 7ca41fdf..5683f913 100644 --- a/lib/python/pyflyby/_autoimp.py +++ b/lib/python/pyflyby/_autoimp.py @@ -1860,7 +1860,7 @@ def auto_import_symbol(fullname, namespaces, db=None, autoimported=None, post_im return True -def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=None): +def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=None, *, extra_db=None): """ Parse ``arg`` for symbols that need to be imported and automatically import them. @@ -1914,6 +1914,8 @@ def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=No if autoimported is None: autoimported = {} db = ImportDB.interpret_arg(db, target_filename=filename) + if extra_db: + db = db|extra_db ok = True for fullname in fullnames: ok &= auto_import_symbol(fullname, namespaces, db, autoimported, post_import_hook=post_import_hook) diff --git a/lib/python/pyflyby/_dynimp.py b/lib/python/pyflyby/_dynimp.py new file mode 100644 index 00000000..69282a9c --- /dev/null +++ b/lib/python/pyflyby/_dynimp.py @@ -0,0 +1,152 @@ +""" +Virtual module to create dynamic import at runtime. + +It is sometime desirable to have auto import which are define only during +a session and never exist on a on-disk file. + +This is injects a Dict module loader as well as a dictionary registry of in +memory module. + +This is mostly use in IPython for lazy variable initialisation without having +to use proxy objects. + +To use, put the following in your IPython startup files +(``~/.ipython/profile_default/startup/autoimp.py`), or in your IPython +configuration file: + + +.. code:: python + + from pyflyby._dynimp import add_import + + add_import("foo", "foo = 1") + + add_import( + "df, data", + ''' + import pandas as pd + data = [1,2,3] + df = pd.DataFrame(data) + ''', + ) + +Now at the IPython prompt, if the pyflyby extension is loaded (either because +you started using the ``py`` cli, or some configuration options like ``ipython +--TerminalIPythonApp.extra_extensions=pyflyby``. When trying to use an undefined +variable like ``foo``, ``df`` or ``data``, the corresponding module will be +executed and the relevant variable imported. + + +""" +import importlib.abc +import importlib.util +import sys + +from textwrap import dedent +from typing import FrozenSet + +from pyflyby._importclns import ImportSet, Import + +module_dict = {} + + +def add_import(names: str, code: str, *, strict: bool = True): + """ + Add a runtime generated import module + + Parameters + ---------- + names: str + name, or comma separated list variable names that should be created by + executing and importing `code`. + code: str + potentially multiline string that will be turned into a module, + executed and from which variables listed in names can be imported. + strict: bool + Raise in case of problem loading IPython of if pyflyby extension not installed. + otherwise just ignore error + + + + Examples + -------- + + >>> add_import('pd, df', ''' + ... import pandas a pd + ... + ... df = pd.DataFrame([[1,2], [3,4]]) + ... ''', strict=False) # don't fail doctest + + """ + try: + ip = _raise_if_problem() + except Exception: + if strict: + raise + else: + return + return _add_import(ip, names, code) + + +def _raise_if_problem(): + try: + import IPython + except ModuleNotFoundError as e: + raise ImportError("Dynamic autoimport requires IPython to be installed") from e + + ip = IPython.get_ipython() + if ip is None: + raise ImportError("Dynamic autoimport only work from within IPython") + + if not hasattr(ip, "_auto_importer"): + raise ValueError( + "IPython needs to be loaded with pyflyby extension for lazy variable to work" + ) + return ip + + +def _add_import(ip, names: str, code: str) -> None: + """ + private version of add_import + """ + assert ip is not None + mang = "pyflyby_autoimport_" + names.replace(",", "_").replace(" ", "_") + a: FrozenSet[Import] = ImportSet(f"from {mang} import {names}")._importset + b: FrozenSet[Import] = ip._auto_importer.db.known_imports._importset + s_import: FrozenSet[Import] = a | b + + ip._auto_importer.db.known_imports = ImportSet._from_imports(list(s_import)) + module_dict[mang] = dedent(code) + +class DictLoader(importlib.abc.Loader): + """ + A dict based loader for in-memory module definition. + """ + def __init__(self, module_name, module_code): + self.module_name = module_name + self.module_code = module_code + + def create_module(self, spec): + return None # Use default module creation semantics + + def exec_module(self, module): + """ + we exec module code directly in memory + """ + exec(self.module_code, module.__dict__) + + +class DictFinder(importlib.abc.MetaPathFinder): + """ + A meta path finder for abode DictLoader + """ + def find_spec(self, fullname, path, target=None): + if fullname in module_dict: + module_code = module_dict[fullname] + loader = DictLoader(fullname, module_code) + return importlib.util.spec_from_loader(fullname, loader) + return None + + +def inject(): + sys.meta_path.insert(0, DictFinder()) diff --git a/lib/python/pyflyby/_importclns.py b/lib/python/pyflyby/_importclns.py index 7eb7a823..b4da8827 100644 --- a/lib/python/pyflyby/_importclns.py +++ b/lib/python/pyflyby/_importclns.py @@ -4,7 +4,7 @@ from __future__ import annotations - +import sys from collections import defaultdict from functools import total_ordering @@ -18,8 +18,18 @@ from pyflyby._util import (cached_attribute, cmp, partition, stable_unique) -from typing import (ClassVar, Dict, FrozenSet, List, - Sequence, Union) +from typing import ( + ClassVar, + Dict, + FrozenSet, + Sequence, + Union, +) + +if sys.version_info < (3, 12): + from typing_extensions import Self +else: + from typing import Self class NoSuchImportError(ValueError): @@ -30,7 +40,7 @@ class ConflictingImportsError(ValueError): pass @total_ordering -class ImportSet(object): +class ImportSet: r""" Representation of a set of imports organized into import statements. @@ -49,8 +59,8 @@ class ImportSet(object): An ``ImportSet`` is an immutable data structure. """ - _EMPTY : ClassVar[ImportSet] - _importset : FrozenSet[Import] + _EMPTY: ClassVar[ImportSet] + _importset: FrozenSet[Import] def __new__(cls, arg, ignore_nonimports=False, ignore_shadowed=False): """ @@ -80,8 +90,11 @@ def __new__(cls, arg, ignore_nonimports=False, ignore_shadowed=False): ignore_nonimports=ignore_nonimports, ignore_shadowed=ignore_shadowed) + def __or__(self: Self, other: Self) -> Self: + return type(self)._from_imports(list(self._importset | other._importset)) + @classmethod - def _from_imports(cls, imports:List[Import], ignore_shadowed:bool=False): + def _from_imports(cls, imports: Sequence[Import], ignore_shadowed: bool = False): """ :type imports: Sequence of `Import` s @@ -114,7 +127,7 @@ def _from_imports(cls, imports:List[Import], ignore_shadowed:bool=False): return self @classmethod - def _from_args(cls, args, ignore_nonimports:bool=False, ignore_shadowed=False): + def _from_args(cls, args, ignore_nonimports:bool=False, ignore_shadowed=False) -> Self: """ :type args: ``tuple`` or ``list`` of `ImportStatement` s, `PythonStatement` s, @@ -133,7 +146,7 @@ def _from_args(cls, args, ignore_nonimports:bool=False, ignore_shadowed=False): # more often. args = [a for a in args if a] if not args: - return cls._EMPTY + return cls._EMPTY # type: ignore[return-value] # If we only got one ``ImportSet``, just return it. if len(args) == 1 and type(args[0]) is cls and not ignore_shadowed: return args[0] @@ -543,9 +556,12 @@ def __new__(cls, arg): if not len(arg): return cls._EMPTY return cls._from_map(arg) - else: - raise TypeError("ImportMap: expected a dict, not a %s" - % (type(arg).__name__,)) + raise TypeError("ImportMap: expected a dict, not a %s" % (type(arg).__name__,)) + + def __or__(self, other): + assert isinstance(other, ImportMap) + assert set(self._data.keys()).intersection(other._data.keys()) == set(), set(self._data.keys()).intersection(other._data.keys()) + return self._merge([self, other]) @classmethod def _from_map(cls, arg): @@ -562,8 +578,8 @@ def _merge(cls, maps): if not maps: return cls._EMPTY data = {} - for map in maps: - data.update(map._data) + for map_ in maps: + data.update(map_._data) return cls(data) def __getitem__(self, k): diff --git a/lib/python/pyflyby/_importdb.py b/lib/python/pyflyby/_importdb.py index a0257cec..8e30708a 100644 --- a/lib/python/pyflyby/_importdb.py +++ b/lib/python/pyflyby/_importdb.py @@ -7,6 +7,10 @@ from collections import defaultdict import os import re +import sys + +from typing import Dict, Any, Tuple + from pyflyby._file import Filename, expand_py_files_from_args, UnsafeFilenameError from pyflyby._idents import dotted_prefixes @@ -16,7 +20,10 @@ from pyflyby._parse import PythonBlock from pyflyby._util import cached_attribute, memoize, stable_unique -from typing import Dict, Any +if sys.version_info <= (3, 12): + from typing_extensions import Self +else: + from typing import Self @memoize @@ -171,7 +178,7 @@ def _expand_tripledots(pathnames, target_dirname): return result -class ImportDB(object): +class ImportDB: """ A database of known, mandatory, canonical imports. @@ -186,6 +193,11 @@ class ImportDB(object): canonical_imports. """ + forget_imports : ImportSet + known_imports : ImportSet + mandatory_imports: ImportSet + canonical_imports: ImportMap + def __new__(cls, *args): if len(args) != 1: raise TypeError @@ -196,6 +208,7 @@ def __new__(cls, *args): return cls._from_data(arg, [], [], []) return cls._from_args(arg) # PythonBlock, Filename, etc + _default_cache: Dict[Any, Any] = {} @@ -366,6 +379,16 @@ def _from_data(cls, known_imports, mandatory_imports, self.canonical_imports = ImportMap(canonical_imports).without_imports(forget_imports) return self + def __or__(self, other:'Self') -> 'Self': + assert isinstance(other, ImportDB) + return self._from_data( + known_imports = self.known_imports | other.known_imports, + mandatory_imports = self.mandatory_imports | other.mandatory_imports, + canonical_imports = self.canonical_imports | other.canonical_imports, + forget_imports = self.forget_imports | other.forget_imports + ) + + @classmethod def _from_args(cls, args): # TODO: support merging input ImportDBs. For now we support @@ -531,7 +554,7 @@ def _parse_import_map(cls, arg): return ImportMap(arg) @cached_attribute - def by_fullname_or_import_as(self): + def by_fullname_or_import_as(self) -> Dict[str, Tuple[Import, ...]]: """ Map from ``fullname`` and ``import_as`` to `Import` s. diff --git a/lib/python/pyflyby/_interactive.py b/lib/python/pyflyby/_interactive.py index eac7ad47..71f1c337 100644 --- a/lib/python/pyflyby/_interactive.py +++ b/lib/python/pyflyby/_interactive.py @@ -15,11 +15,14 @@ import subprocess import sys +from typing import List, Any, Dict + from pyflyby._autoimp import (LoadSymbolError, ScopeStack, auto_eval, auto_import, clear_failed_imports_cache, load_symbol) +from pyflyby._dynimp import inject as inject_dynamic_import from pyflyby._comms import (initialize_comms, remove_comms, send_comm_message, MISSING_IMPORTS) from pyflyby._file import Filename, atomic_write_file, read_file @@ -1368,20 +1371,30 @@ def UpdateIPythonStdioCtx(): -class _EnableState(object): +class _EnableState: DISABLING = "DISABLING" DISABLED = "DISABLED" ENABLING = "ENABLING" ENABLED = "ENABLED" -class AutoImporter(object): +class AutoImporter: """ Auto importer enable state. The state is attached to an IPython "application". """ + db: ImportDB + app: Any + _state: _EnableState + _disablers: List[Any] + + _errored: bool + _ip: Any + _ast_transformer: Any + _autoimported_this_cell: Dict[Any, Any] + def __new__(cls, arg=Ellipsis): """ Get the AutoImporter for the given app, or create and assign one. @@ -1418,7 +1431,7 @@ def __new__(cls, arg=Ellipsis): raise TypeError("AutoImporter(): unexpected %s" % (clsname,)) @classmethod - def _from_app(cls, app): + def _from_app(cls, app) -> 'AutoImporter': subapp = getattr(app, "subapp", None) if subapp is not None: app = subapp @@ -1432,6 +1445,7 @@ def _from_app(cls, app): # Create a new instance and assign to the app. self = cls._construct(app) app.auto_importer = self + self.db = ImportDB("") return self @classmethod @@ -2464,7 +2478,8 @@ def post_import_hook(imp): send_comm_message(MISSING_IMPORTS, {"missing_imports": str(imp)}) return self._safe_call( - auto_import, arg, namespaces, + auto_import, arg=arg, namespaces=namespaces, + extra_db=self.db, autoimported=self._autoimported_this_cell, raise_on_error=raise_on_error, on_error=on_error, post_import_hook=post_import_hook) @@ -2567,6 +2582,8 @@ def load_ipython_extension(arg=Ellipsis): os.path.dirname(__file__)) # Turn on the auto-importer. auto_importer = AutoImporter(arg) + if arg is not Ellipsis: + arg._auto_importer = auto_importer auto_importer.enable(even_if_previously_errored=True) # Clear ImportDB cache. ImportDB.clear_default_cache() @@ -2584,6 +2601,7 @@ def load_ipython_extension(arg=Ellipsis): enable_signal_handler_debugger() enable_sigterm_handler(on_existing_handler='keep_existing') add_debug_functions_to_builtins() + inject_dynamic_import() initialize_comms() diff --git a/setup.py b/setup.py index 8063f968..9b8c179d 100755 --- a/setup.py +++ b/setup.py @@ -221,7 +221,12 @@ def make_distribution(self): "License :: OSI Approved :: MIT License", "Programming Language :: Python", ], - install_requires=["six", "toml", "black"], + install_requires=[ + "six", + "toml", + "black", + "typing_extensions>=4.6; python_version<'3.12'" + ], python_requires=">3.8, <4", tests_require=['pexpect>=3.3', 'pytest', 'epydoc', 'rlipython', 'requests'], cmdclass = { diff --git a/tests/test_importclns.py b/tests/test_importclns.py index 7e98837d..86b5c295 100644 --- a/tests/test_importclns.py +++ b/tests/test_importclns.py @@ -241,3 +241,9 @@ def test_ImportSet_without_imports_star_dot_1(): def test_ImportMap_1(): importmap = ImportMap({'a.b': 'aa.bb', 'a.b.c': 'aa.bb.cc'}) assert importmap['a.b'] == 'aa.bb' + +def test_ImportSet_union(): + a = ImportSet('from numpy import einsum, cos') + b = ImportSet('from numpy import sin, cos') + c = ImportSet('from numpy import einsum, sin, cos') + assert a|b == c