From 162f729f046e5aa61495de7da29d7cffb29f5ca7 Mon Sep 17 00:00:00 2001 From: rocky Date: Sun, 25 Jun 2023 12:58:49 -0400 Subject: [PATCH] Move more code out of mathics.builtin.__init__ Moved to mathics.eval.builtin. Possibly not the best place, but it is better and is a start. --- mathics/builtin/__init__.py | 171 +---------------------------- mathics/builtin/box/layout.py | 5 +- mathics/builtin/lowlevelprofile.py | 4 +- mathics/core/definitions.py | 5 +- mathics/docpipeline.py | 10 +- mathics/eval/builtin.py | 167 ++++++++++++++++++++++++++++ mathics/eval/makeboxes.py | 4 +- mathics/eval/pymathics.py | 4 +- 8 files changed, 191 insertions(+), 179 deletions(-) create mode 100755 mathics/eval/builtin.py diff --git a/mathics/builtin/__init__.py b/mathics/builtin/__init__.py index 35fb603e3..25c9e4ee7 100755 --- a/mathics/builtin/__init__.py +++ b/mathics/builtin/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- """ -Mathics Builtin Functions and Variables. +Mathics Builtin Functions and Variables. Mathics has over a thousand Built-in functions and variables, all of which are defined here. @@ -10,10 +10,6 @@ ``mathics.format`` for rendering details, or ``mathics.compile`` for compilation details. -What remains here is then mostly the top-level definition of a Mathics -Builtin, and attributes that have not been segregated elsewhere such -as has been done for one of the other modules listed above. - A Mathics Builtin is implemented one of a particular kind of Python class. Within these classes class variables give properties of the builtin class such as the Builtin's Attributes, its Information text, @@ -22,24 +18,13 @@ import glob import importlib -import inspect import os.path as osp import pkgutil import re -from typing import List, Optional -from mathics.builtin.base import ( - Builtin, - Operator, - PatternObject, - SympyObject, - mathics_to_python, -) -from mathics.core.pattern import pattern_objects -from mathics.core.symbols import Symbol -from mathics.eval.makeboxes import builtins_precedence +from mathics.builtin.base import Builtin +from mathics.eval.builtin import add_builtins, import_builtins, name_is_builtin_symbol from mathics.settings import ENABLE_FILES_MODULE -from mathics.version import __version__ # noqa used in loading to check consistency. # Get a list of files in this directory. We'll exclude from the start # files with leading characters we don't want like __init__ with its leading underscore. @@ -48,146 +33,6 @@ for f in glob.glob(osp.join(osp.dirname(__file__), "[a-z]*.py")) ] - -def add_builtins(new_builtins): - from mathics.core.convert.sympy import mathics_to_sympy, sympy_to_mathics - - for var_name, builtin in new_builtins: - name = builtin.get_name() - if hasattr(builtin, "python_equivalent"): - # print("XXX0", builtin.python_equivalent) - mathics_to_python[name] = builtin.python_equivalent - - if isinstance(builtin, SympyObject): - mathics_to_sympy[name] = builtin - for sympy_name in builtin.get_sympy_names(): - # print("XXX1", sympy_name) - sympy_to_mathics[sympy_name] = builtin - if isinstance(builtin, Operator): - builtins_precedence[Symbol(name)] = builtin.precedence - if isinstance(builtin, PatternObject): - pattern_objects[name] = builtin.__class__ - _builtins.update(dict(new_builtins)) - - -def builtins_dict(): - return { - builtin.get_name(): builtin - for modname, builtins in builtins_by_module.items() - for builtin in builtins - } - - -def contribute(definitions): - # let MakeBoxes contribute first - _builtins["System`MakeBoxes"].contribute(definitions) - for name, item in _builtins.items(): - if name != "System`MakeBoxes": - item.contribute(definitions) - - from mathics.core.definitions import Definition - from mathics.core.expression import ensure_context - from mathics.core.parser import all_operator_names - - # All builtins are loaded. Create dummy builtin definitions for - # any remaining operators that don't have them. This allows - # operators like \[Cup] to behave correctly. - for operator in all_operator_names: - if not definitions.have_definition(ensure_context(operator)): - op = ensure_context(operator) - definitions.builtin[op] = Definition(name=op) - - -def import_builtins(module_names: List[str], submodule_name=None) -> None: - """ - Imports the list of Mathics Built-in modules so that inside - Mathics we have these Builtin Functions, like Plus[], List[] are defined. - - """ - - def import_module(module_name: str, import_name: str): - try: - module = importlib.import_module(import_name) - except Exception as e: - print(e) - print(f" Not able to load {module_name}. Check your installation.") - print(f" mathics.builtin loads from {__file__[:-11]}") - return None - - if module: - modules.append(module) - - if submodule_name: - import_module(submodule_name, f"mathics.builtin.{submodule_name}") - - for module_name in module_names: - import_name = ( - f"mathics.builtin.{submodule_name}.{module_name}" - if submodule_name - else f"mathics.builtin.{module_name}" - ) - import_module(module_name, import_name) - - -def name_is_builtin_symbol(module, name: str) -> Optional[type]: - """ - Checks if ``name`` should be added to definitions, and return - its associated Builtin class. - - Return ``None`` if the name should not get added to definitions. - """ - if name.startswith("_"): - return None - - module_object = getattr(module, name) - - # Look only at Class objects. - if not inspect.isclass(module_object): - return None - - # Skip those builtins defined in or imported from another module. - - # rocky: I think this is a code smell. It doesn't feel like - # we should have to do this if things are organized and modularized - # builtins and use less custom code. - # mmatera reports that we need this because of the interaction of - # * the custom Mathics3 loading/importing mechanism, - # * the builtin module hierarchy, e.g. mathics.builtin.arithmetic - # nested under mathics.builtin, and - # * our custom doc/doctest and possibly custom checking system - - # Mathics3 modules modules, however, right now import all builtin modules from - # __init__ - # Note Mathics3 modules do not support buitin hierarchies, e.g. - # pymathics.graph.parametric is allowed but not pymathics.graph.parametric.xxx. - # This too has to do with the custom doc/doctest that is currently used. - - if inspect.getmodule( - module_object - ) is not module and not module.__name__.startswith("pymathics."): - return None - - # Skip objects in module mathics.builtin.base. - if module_object.__module__ == "mathics.builtin.base": - return None - - # Skip those builtins that are not submodules of mathics.builtin. - if not ( - module_object.__module__.startswith("mathics.builtin.") - or module_object.__module__.startswith("pymathics.") - ): - return None - - # If it is not a subclass of Builtin, skip it. - if not issubclass(module_object, Builtin): - return None - - # Skip Builtin classes that were explicitly marked for skipping. - if module_object in getattr(module, "DOES_NOT_ADD_BUILTIN_DEFINITION", []): - return None - return module_object - - # FIXME: redo using importlib since that is probably less fragile. exclude_files = {"codetables", "base"} module_names = [ @@ -195,7 +40,7 @@ def name_is_builtin_symbol(module, name: str) -> Optional[type]: ] modules = [] -import_builtins(module_names) +import_builtins(modules, module_names) _builtins_list = [] builtins_by_module = {} @@ -237,11 +82,10 @@ def name_is_builtin_symbol(module, name: str) -> Optional[type]: builtin_module = importlib.import_module(import_name) submodule_names = [ - modname - for importer, modname, ispkg in pkgutil.iter_modules(builtin_module.__path__) + modname for _, modname, _ in pkgutil.iter_modules(builtin_module.__path__) ] # print("XXX3", submodule_names) - import_builtins(submodule_names, subdir) + import_builtins(modules, submodule_names, subdir) for module in modules: builtins_by_module[module.__name__] = [] @@ -262,9 +106,6 @@ def name_is_builtin_symbol(module, name: str) -> Optional[type]: new_builtins = _builtins_list -# FIXME: some magic is going on here.. -_builtins = {} - add_builtins(new_builtins) display_operators_set = set() diff --git a/mathics/builtin/box/layout.py b/mathics/builtin/box/layout.py index 829b929e3..eb7b55e8d 100644 --- a/mathics/builtin/box/layout.py +++ b/mathics/builtin/box/layout.py @@ -7,8 +7,6 @@ At the other end, the top level, we have a Notebook which is just a \ collection of Expressions usually contained in boxes. """ -# Docs are not yet ready for prime time. Maybe after release 6.0.0. -no_doc = True from mathics.builtin.base import Builtin from mathics.builtin.box.expression import BoxExpression @@ -32,6 +30,9 @@ ) from mathics.eval.makeboxes import eval_makeboxes +# Docs are not yet ready for prime time. Maybe after release 6.0.0. +no_doc = True + def to_boxes(x, evaluation: Evaluation, options={}) -> BoxElementMixin: """ diff --git a/mathics/builtin/lowlevelprofile.py b/mathics/builtin/lowlevelprofile.py index 6292a6779..033a2d4f4 100644 --- a/mathics/builtin/lowlevelprofile.py +++ b/mathics/builtin/lowlevelprofile.py @@ -3,7 +3,7 @@ """ Low-level Profiling -Low-level (Python) profile from inside the Mathics interpreter +Low-level (Python) profile from inside the Mathics3 interpreter """ @@ -12,7 +12,7 @@ import sys from io import StringIO -from mathics.builtin import Builtin +from mathics.builtin.base import Builtin from mathics.core.attributes import A_HOLD_ALL_COMPLETE, A_PROTECTED from mathics.core.convert.python import from_python from mathics.core.evaluation import Evaluation diff --git a/mathics/core/definitions.py b/mathics/core/definitions.py index 2529fc63f..64de0cb91 100644 --- a/mathics/core/definitions.py +++ b/mathics/core/definitions.py @@ -17,6 +17,7 @@ from mathics.core.expression import Expression from mathics.core.symbols import Atom, Symbol, strip_context from mathics.core.systemsymbols import SymbolGet +from mathics.eval.builtin import definition_contribute type_compiled_pattern = type(re.compile("a.a")) @@ -137,7 +138,7 @@ def __init__( self.timing_trace_evaluation = False if add_builtin: - from mathics.builtin import contribute, modules + from mathics.builtin import modules from mathics.settings import ROOT_DIR loaded = False @@ -149,7 +150,7 @@ def __init__( self.builtin = pickle.load(builtin_file) loaded = True if not loaded: - contribute(self) + definition_contribute(self) for module in extension_modules: try: load_pymathics_module(self, module) diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py index 5fc005940..c5132fa77 100644 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -5,7 +5,8 @@ Does 2 things which can either be done independently or as a pipeline: -1. Extracts tests and runs them from static mdoc files and docstrings from Mathics built-in functions +1. Extracts tests and runs them from static mdoc files and docstrings from Mathics + built-in functions 2. Creates/updates internal documentation data """ @@ -21,19 +22,20 @@ import mathics import mathics.settings from mathics import settings, version_string -from mathics.builtin import builtins_dict +from mathics.builtin import builtins_by_module from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation, Output from mathics.core.parser import MathicsSingleLineFeeder from mathics.doc.common_doc import MathicsMainDocumentation +from mathics.eval.builtin import builtins_dict from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule from mathics.timing import show_lru_cache_statistics -builtins = builtins_dict() +builtins = builtins_dict(builtins_by_module) class TestOutput(Output): - def max_stored_size(self, settings): + def max_stored_size(self, _): return None diff --git a/mathics/eval/builtin.py b/mathics/eval/builtin.py new file mode 100755 index 000000000..3766f5bb4 --- /dev/null +++ b/mathics/eval/builtin.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +""" +Code around loading Mathics3 Builtin Functions and Variables. + +This code loads the top-level definition of a Mathics3 +Builtin, and attributes that have not been segregated elsewhere such +as has been done for one of the other modules listed above. +""" + +import importlib +import inspect +from typing import Optional + +from mathics.core.pattern import pattern_objects +from mathics.core.symbols import Symbol +from mathics.eval.makeboxes import builtins_precedence + +_builtins = {} + +# The fact that are importing inside here, suggests add_builtins +# should get moved elsewhere. +def add_builtins(new_builtins): + from mathics.builtin.base import ( + Operator, + PatternObject, + SympyObject, + mathics_to_python, + ) + from mathics.core.convert.sympy import mathics_to_sympy, sympy_to_mathics + + for _, builtin in new_builtins: + name = builtin.get_name() + if hasattr(builtin, "python_equivalent"): + # print("XXX0", builtin.python_equivalent) + mathics_to_python[name] = builtin.python_equivalent + + if isinstance(builtin, SympyObject): + mathics_to_sympy[name] = builtin + for sympy_name in builtin.get_sympy_names(): + # print("XXX1", sympy_name) + sympy_to_mathics[sympy_name] = builtin + if isinstance(builtin, Operator): + builtins_precedence[Symbol(name)] = builtin.precedence + if isinstance(builtin, PatternObject): + pattern_objects[name] = builtin.__class__ + _builtins.update(dict(new_builtins)) + + +def builtins_dict(builtins_by_module): + return { + builtin.get_name(): builtin + for _, builtins in builtins_by_module.items() + for builtin in builtins + } + + +def definition_contribute(definitions): + # let MakeBoxes contribute first + _builtins["System`MakeBoxes"].contribute(definitions) + for name, item in _builtins.items(): + if name != "System`MakeBoxes": + item.contribute(definitions) + + from mathics.core.definitions import Definition + from mathics.core.expression import ensure_context + from mathics.core.parser import all_operator_names + + # All builtins are loaded. Create dummy builtin definitions for + # any remaining operators that don't have them. This allows + # operators like \[Cup] to behave correctly. + for operator in all_operator_names: + if not definitions.have_definition(ensure_context(operator)): + op = ensure_context(operator) + definitions.builtin[op] = Definition(name=op) + + +# TODO: When we drop Python 3.7, +# module_names can be a List[Literal] +def import_builtins(modules: list, module_names: list, submodule_name=None) -> None: + """ + Imports the list of Mathics3 Built-in modules so that inside + Mathics3 Builtin Functions, like Plus[], List[] are defined. + """ + + def import_module(module_name: str, import_name: str): + try: + module = importlib.import_module(import_name) + except Exception as e: + print(e) + print(f" Not able to load {module_name}. Check your installation.") + print(f" mathics.builtin loads from {__file__[:-11]}") + return None + + if module: + modules.append(module) + + if submodule_name: + import_module(submodule_name, f"mathics.builtin.{submodule_name}") + + for module_name in module_names: + import_name = ( + f"mathics.builtin.{submodule_name}.{module_name}" + if submodule_name + else f"mathics.builtin.{module_name}" + ) + import_module(module_name, import_name) + + +def name_is_builtin_symbol(module, name: str) -> Optional[type]: + """ + Checks if ``name`` should be added to definitions, and return + its associated Builtin class. + + Return ``None`` if the name should not get added to definitions. + """ + if name.startswith("_"): + return None + + module_object = getattr(module, name) + + # Look only at Class objects. + if not inspect.isclass(module_object): + return None + + # Skip those builtins defined in or imported from another module. + + # rocky: I think this is a code smell. It doesn't feel like + # we should have to do this if things are organized and modularized + # builtins and use less custom code. + # mmatera reports that we need this because of the interaction of + # * the custom Mathics3 loading/importing mechanism, + # * the builtin module hierarchy, e.g. mathics.builtin.arithmetic + # nested under mathics.builtin, and + # * our custom doc/doctest and possibly custom checking system + + # Mathics3 modules modules, however, right now import all builtin modules from + # __init__ + # Note Mathics3 modules do not support buitin hierarchies, e.g. + # pymathics.graph.parametric is allowed but not pymathics.graph.parametric.xxx. + # This too has to do with the custom doc/doctest that is currently used. + + if inspect.getmodule( + module_object + ) is not module and not module.__name__.startswith("pymathics."): + return None + + # Skip objects in module mathics.builtin.base. + if module_object.__module__ == "mathics.builtin.base": + return None + + # Skip those builtins that are not submodules of mathics.builtin. + if not ( + module_object.__module__.startswith("mathics.builtin.") + or module_object.__module__.startswith("pymathics.") + ): + return None + + from mathics.builtin.base import Builtin + + # If it is not a subclass of Builtin, skip it. + if not issubclass(module_object, Builtin): + return None + + # Skip Builtin classes that were explicitly marked for skipping. + if module_object in getattr(module, "DOES_NOT_ADD_BUILTIN_DEFINITION", []): + return None + return module_object diff --git a/mathics/eval/makeboxes.py b/mathics/eval/makeboxes.py index 52665c5e0..5ecd4e65e 100644 --- a/mathics/eval/makeboxes.py +++ b/mathics/eval/makeboxes.py @@ -11,7 +11,6 @@ from mathics.core.atoms import Complex, Integer, Rational, Real, String, SymbolI from mathics.core.convert.expression import to_expression_with_specialization -from mathics.core.definitions import OutputForms from mathics.core.element import BaseElement, BoxElementMixin, EvalMixin from mathics.core.evaluation import Evaluation from mathics.core.expression import Expression @@ -36,7 +35,6 @@ from mathics.core.systemsymbols import ( SymbolComplex, SymbolMinus, - SymbolOutputForm, SymbolRational, SymbolRowBox, SymbolStandardForm, @@ -131,6 +129,8 @@ def do_format_element( superfluous enclosing formats. """ + from mathics.core.definitions import OutputForms + evaluation.inc_recursion_depth() try: expr = element diff --git a/mathics/eval/pymathics.py b/mathics/eval/pymathics.py index 6cc920238..448d761a6 100644 --- a/mathics/eval/pymathics.py +++ b/mathics/eval/pymathics.py @@ -6,9 +6,10 @@ import inspect import sys -from mathics.builtin import name_is_builtin_symbol +from mathics.builtin import builtins_by_module from mathics.builtin.base import Builtin from mathics.core.definitions import Definitions +from mathics.eval.builtin import name_is_builtin_symbol # The below set and dictionary are used in document generation # for Pymathics modules. @@ -48,7 +49,6 @@ def load_pymathics_module(definitions, module_name: str): Loads Mathics builtin objects and their definitions from an external Python module in the pymathics module namespace. """ - from mathics.builtin import Builtin, builtins_by_module, name_is_builtin_symbol if module_name in sys.modules: loaded_module = importlib.reload(sys.modules[module_name])