Add infrastructure to lazily define variable via autoimport

This should cover the feature request on IPython to have the ability to
lazily define variable that can take some time to be created when first
access in IPython;

It does so by utilizing the autoimport machinery of pyflyby and creating
runtime modules via a import hook.

This has many advantage:
 - we do not use a proxy object.
 - we do not have to deal with multiple variable being potentially
   defined together and thik of re-excution.

.github/workflows/lint.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.12"]
 
     steps:
     - uses: actions/checkout@v3

README.rst

@@ -81,6 +81,36 @@ or
    [PYFLYBY] from base64 import b64decode
    Out[1]: 'hello'
 
+Auto importer lazy variables
+----------------------------
+
+It is possible to use the autoimporter to lazily define variables.
+
+To use, put the following in your IPython startup files
+(``~/.ipython/profile_default/startup/autoimp.py``), or in your IPython
+configuration file:
+
+.. code:: python
+
+   
+    from pyflyby import add_import
+
+    add_import("foo", "foo = 1")
+
+    add_import(
+        "df, data as dd",
+        '''
+        import pandas as pd
+        data = [1,2,3]
+        df =  pd.DataFrame(data)
+    ''')
+
+
+You can add the keyword ``strict=False`` to not fail if not in IPython or of the
+pyflyby extensions is not loaded.
+
+
+
 
 Quick start: ``py`` command-line multi-tool
 ===========================================

lib/python/pyflyby/__init__.py

@@ -6,6 +6,7 @@
 
 from   pyflyby._autoimp         import (auto_eval, auto_import,
                                         find_missing_imports)
+from   pyflyby._dynimp          import add_import
 from   pyflyby._dbg             import (add_debug_functions_to_builtins,
                                         attach_debugger, debug_on_exception,
                                         debug_statement, debugger,

lib/python/pyflyby/_autoimp.py

@@ -1860,7 +1860,7 @@ def auto_import_symbol(fullname, namespaces, db=None, autoimported=None, post_im
     return True
 
 
-def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=None):
+def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=None, *, extra_db=None):
     """
     Parse ``arg`` for symbols that need to be imported and automatically import
     them.
@@ -1914,6 +1914,8 @@ def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=No
     if autoimported is None:
         autoimported = {}
     db = ImportDB.interpret_arg(db, target_filename=filename)
+    if extra_db:
+        db = db|extra_db
     ok = True
     for fullname in fullnames:
         ok &= auto_import_symbol(fullname, namespaces, db, autoimported, post_import_hook=post_import_hook)

lib/python/pyflyby/_dynimp.py

@@ -0,0 +1,152 @@
+"""
+Virtual module to create dynamic import at runtime.
+
+It is sometime desirable to have auto import which are define only during
+a session and never exist on a on-disk file.
+
+This is injects a Dict module loader as well as a dictionary registry of in
+memory module.
+
+This is mostly use in IPython for lazy variable initialisation without having
+to use proxy objects.
+
+To use, put the following in your IPython startup files
+(``~/.ipython/profile_default/startup/autoimp.py`), or in your IPython
+configuration file:
+
+
+.. code:: python
+
+    from pyflyby._dynimp import add_import
+
+    add_import("foo", "foo = 1")
+
+    add_import(
+        "df, data",
+        '''
+        import pandas as pd
+        data = [1,2,3]
+        df =  pd.DataFrame(data)
+    ''',
+    )
+
+Now at the IPython prompt, if the pyflyby extension is loaded (either because
+you started  using the ``py`` cli, or some configuration options like  ``ipython
+--TerminalIPythonApp.extra_extensions=pyflyby``. When trying to use an undefined
+variable like ``foo``, ``df`` or ``data``, the corresponding module will be
+executed and the relevant variable imported.
+
+
+"""
+import importlib.abc
+import importlib.util
+import sys
+
+from textwrap import dedent
+from typing import FrozenSet
+
+from pyflyby._importclns import ImportSet, Import
+
+module_dict = {}
+
+
+def add_import(names: str, code: str, *, strict: bool = True):
+    """
+    Add a runtime generated import module
+
+    Parameters
+    ----------
+    names: str
+        name, or comma separated list variable names that should be created by
+        executing and importing `code`.
+    code: str
+        potentially multiline string that will be turned into a module,
+        executed and from which variables listed in names can be imported.
+    strict: bool
+        Raise in case of problem loading IPython of if pyflyby extension not installed.
+        otherwise just ignore error
+
+
+
+    Examples
+    --------
+
+    >>> add_import('pd, df', '''
+    ...     import pandas a pd
+    ...
+    ...     df = pd.DataFrame([[1,2], [3,4]])
+    ... ''', strict=False)  # don't fail doctest
+
+    """
+    try:
+        ip = _raise_if_problem()
+    except Exception:
+        if strict:
+            raise
+        else:
+            return
+    return _add_import(ip, names, code)
+
+
+def _raise_if_problem():
+    try:
+        import IPython
+    except ModuleNotFoundError as e:
+        raise ImportError("Dynamic autoimport requires IPython to be installed") from e
+
+    ip = IPython.get_ipython()
+    if ip is None:
+        raise ImportError("Dynamic autoimport only work from within IPython")
+
+    if not hasattr(ip, "_auto_importer"):
+        raise ValueError(
+            "IPython needs to be loaded with pyflyby extension for lazy variable to work"
+        )
+    return ip
+
+
+def _add_import(ip, names: str, code: str) -> None:
+    """
+    private version of add_import
+    """
+    assert ip is not None
+    mang = "pyflyby_autoimport_" + names.replace(",", "_").replace(" ", "_")
+    a: FrozenSet[Import] = ImportSet(f"from {mang} import {names}")._importset
+    b: FrozenSet[Import] = ip._auto_importer.db.known_imports._importset
+    s_import: FrozenSet[Import] = a | b
+
+    ip._auto_importer.db.known_imports = ImportSet._from_imports(list(s_import))
+    module_dict[mang] = dedent(code)
+
+class DictLoader(importlib.abc.Loader):
+    """
+    A dict based loader for in-memory module definition.
+    """
+    def __init__(self, module_name, module_code):
+        self.module_name = module_name
+        self.module_code = module_code
+
+    def create_module(self, spec):
+        return None  # Use default module creation semantics
+
+    def exec_module(self, module):
+        """
+        we exec module code directly in memory
+        """
+        exec(self.module_code, module.__dict__)
+
+
+class DictFinder(importlib.abc.MetaPathFinder):
+    """
+    A meta path finder for abode DictLoader
+    """
+    def find_spec(self, fullname, path, target=None):
+        if fullname in module_dict:
+            module_code = module_dict[fullname]
+            loader = DictLoader(fullname, module_code)
+            return importlib.util.spec_from_loader(fullname, loader)
+        return None
+
+
+def inject():
+    sys.meta_path.insert(0, DictFinder())

lib/python/pyflyby/_importclns.py

@@ -4,7 +4,7 @@
 
 from __future__ import annotations
 
-
+import sys
 
 from   collections              import defaultdict
 from   functools                import total_ordering
@@ -18,8 +18,18 @@
 from   pyflyby._util            import (cached_attribute, cmp, partition,
                                         stable_unique)
 
-from   typing                   import (ClassVar, Dict, FrozenSet, List,
-                                        Sequence, Union)
+from typing import (
+    ClassVar,
+    Dict,
+    FrozenSet,
+    Sequence,
+    Union,
+)
+
+if sys.version_info < (3, 12):
+    from typing_extensions          import Self
+else:
+    from typing import Self
 
 
 class NoSuchImportError(ValueError):
@@ -30,7 +40,7 @@ class ConflictingImportsError(ValueError):
     pass
 
 @total_ordering
-class ImportSet(object):
+class ImportSet:
     r"""
     Representation of a set of imports organized into import statements.
 
@@ -49,8 +59,8 @@ class ImportSet(object):
     An ``ImportSet`` is an immutable data structure.
     """
 
-    _EMPTY : ClassVar[ImportSet]
-    _importset : FrozenSet[Import]
+    _EMPTY: ClassVar[ImportSet]
+    _importset: FrozenSet[Import]
 
     def __new__(cls, arg, ignore_nonimports=False, ignore_shadowed=False):
         """
@@ -80,8 +90,11 @@ def __new__(cls, arg, ignore_nonimports=False, ignore_shadowed=False):
             ignore_nonimports=ignore_nonimports,
             ignore_shadowed=ignore_shadowed)
 
+    def __or__(self: Self, other: Self) -> Self:
+        return type(self)._from_imports(list(self._importset | other._importset))
+
     @classmethod
-    def _from_imports(cls, imports:List[Import], ignore_shadowed:bool=False):
+    def _from_imports(cls, imports: Sequence[Import], ignore_shadowed: bool = False):
         """
         :type imports:
           Sequence of `Import` s
@@ -114,7 +127,7 @@ def _from_imports(cls, imports:List[Import], ignore_shadowed:bool=False):
         return self
 
     @classmethod
-    def _from_args(cls, args, ignore_nonimports:bool=False, ignore_shadowed=False):
+    def _from_args(cls, args, ignore_nonimports:bool=False, ignore_shadowed=False) -> Self:
         """
         :type args:
           ``tuple`` or ``list`` of `ImportStatement` s, `PythonStatement` s,
@@ -133,7 +146,7 @@ def _from_args(cls, args, ignore_nonimports:bool=False, ignore_shadowed=False):
         # more often.
         args = [a for a in args if a]
         if not args:
-            return cls._EMPTY
+            return cls._EMPTY  # type: ignore[return-value]
         # If we only got one ``ImportSet``, just return it.
         if len(args) == 1 and type(args[0]) is cls and not ignore_shadowed:
             return args[0]
@@ -543,9 +556,12 @@ def __new__(cls, arg):
             if not len(arg):
                 return cls._EMPTY
             return cls._from_map(arg)
-        else:
-            raise TypeError("ImportMap: expected a dict, not a %s"
-                            % (type(arg).__name__,))
+        raise TypeError("ImportMap: expected a dict, not a %s" % (type(arg).__name__,))
+
+    def __or__(self, other):
+        assert isinstance(other, ImportMap)
+        assert set(self._data.keys()).intersection(other._data.keys()) == set(), set(self._data.keys()).intersection(other._data.keys())
+        return self._merge([self, other])
 
     @classmethod
     def _from_map(cls, arg):
@@ -562,8 +578,8 @@ def _merge(cls, maps):
         if not maps:
             return cls._EMPTY
         data = {}
-        for map in maps:
-            data.update(map._data)
+        for map_ in maps:
+            data.update(map_._data)
         return cls(data)
 
     def __getitem__(self, k):