diff --git a/admin-tools/build_and_check_manifest.py b/admin-tools/build_and_check_manifest.py index 9057a0f34..0feb64af9 100755 --- a/admin-tools/build_and_check_manifest.py +++ b/admin-tools/build_and_check_manifest.py @@ -2,7 +2,14 @@ import sys -from mathics.builtin import Builtin, modules, name_is_builtin_symbol +from mathics.builtin.base import Builtin +from mathics.core.load_builtin import ( + import_and_load_builtins, + modules, + name_is_builtin_symbol, +) + +import_and_load_builtins() def generate_available_builtins_names(): diff --git a/examples/symbolic_logic/gries_schneider/test_gs.py b/examples/symbolic_logic/gries_schneider/test_gs.py index 7212ec8ee..19cd96d03 100644 --- a/examples/symbolic_logic/gries_schneider/test_gs.py +++ b/examples/symbolic_logic/gries_schneider/test_gs.py @@ -4,8 +4,10 @@ from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation +from mathics.core.load_builtin import import_and_load_builtins from mathics.core.parser import MathicsSingleLineFeeder, parse +import_and_load_builtins() definitions = Definitions(add_builtin=True) for i in range(0, 4): diff --git a/mathics/builtin/__init__.py b/mathics/builtin/__init__.py index 946d024ba..efb232604 100755 --- a/mathics/builtin/__init__.py +++ b/mathics/builtin/__init__.py @@ -15,38 +15,3 @@ builtin class such as the Builtin's Attributes, its Information text, among other things. """ - -import os -import os.path as osp - -from mathics.core.load_builtin import ( - add_builtins_from_builtin_modules, - get_module_names, - import_builtin_subdirectories, - import_builtins, - initialize_display_operators_set, -) -from mathics.settings import ENABLE_FILES_MODULE - -# Get import modules in this directory of Python modules that contain -# Mathics3 Builtin class definitions. - -builtin_path = osp.dirname(__file__) -exclude_files = {"codetables", "base"} -module_names = get_module_names(builtin_path, exclude_files) -modules = [] -import_builtins(module_names, modules) - -# Get import modules in subdirectories of this directory of Python -# modules that contain Mathics3 Builtin class definitions. - -# The files_io module handles local file access, reading and writing.. -# In some sandboxed settings, such as running Mathics from as a remote -# server, we disallow local file access. -disable_file_module_names = set() if ENABLE_FILES_MODULE else {"files_io"} - -subdirectories = next(os.walk(builtin_path))[1] -import_builtin_subdirectories(subdirectories, disable_file_module_names, modules) - -add_builtins_from_builtin_modules(modules) -initialize_display_operators_set() diff --git a/mathics/core/definitions.py b/mathics/core/definitions.py index 93643fcc0..bd2d8d3bf 100644 --- a/mathics/core/definitions.py +++ b/mathics/core/definitions.py @@ -15,9 +15,10 @@ from mathics.core.convert.expression import to_mathics_list from mathics.core.element import fully_qualified_symbol_name from mathics.core.expression import Expression -from mathics.core.load_builtin import definition_contribute +from mathics.core.load_builtin import definition_contribute, mathics3_builtins_modules from mathics.core.symbols import Atom, Symbol, strip_context from mathics.core.systemsymbols import SymbolGet +from mathics.settings import ROOT_DIR type_compiled_pattern = type(re.compile("a.a")) @@ -138,12 +139,12 @@ def __init__( self.timing_trace_evaluation = False if add_builtin: - from mathics.builtin import modules - from mathics.settings import ROOT_DIR - loaded = False if builtin_filename is not None: - builtin_dates = [get_file_time(module.__file__) for module in modules] + builtin_dates = [ + get_file_time(module.__file__) + for module in mathics3_builtins_modules + ] builtin_time = max(builtin_dates) if get_file_time(builtin_filename) > builtin_time: builtin_file = open(builtin_filename, "rb") @@ -770,7 +771,6 @@ def __init__( builtin=None, is_numeric=False, ) -> None: - super(Definition, self).__init__() self.name = name diff --git a/mathics/core/load_builtin.py b/mathics/core/load_builtin.py index 376412412..a2a3453e2 100755 --- a/mathics/core/load_builtin.py +++ b/mathics/core/load_builtin.py @@ -8,19 +8,29 @@ import importlib import inspect +import os import os.path as osp import pkgutil from glob import glob +from types import ModuleType from typing import List, Optional from mathics.core.pattern import pattern_objects from mathics.core.symbols import Symbol from mathics.eval.makeboxes import builtins_precedence +from mathics.settings import ENABLE_FILES_MODULE + +# List of Python modules contain Mathics3 Builtins. +# This list used outside to gather documentation, +# and test module consistency. It is +# is initialized via below import_builtins modules +mathics3_builtins_modules: List[ModuleType] = [] _builtins = {} builtins_by_module = {} display_operators_set = set() + # The fact that are importing inside here, suggests add_builtins # should get moved elsewhere. def add_builtins(new_builtins): @@ -50,7 +60,7 @@ def add_builtins(new_builtins): _builtins.update(dict(new_builtins)) -def add_builtins_from_builtin_modules(modules): +def add_builtins_from_builtin_modules(modules: List[ModuleType]): # This can be put at the top after mathics.builtin.__init__ # cleanup is done. from mathics.builtin.base import Builtin @@ -110,10 +120,44 @@ def get_module_names(builtin_path: str, exclude_files: set) -> list: return [f for f in py_files if f not in exclude_files] +def import_and_load_builtins(): + """ + Imports Builtin modules in mathics.builtin and add rules, and definitions from that. + """ + builtin_path = osp.join( + osp.dirname( + __file__, + ), + "..", + "builtin", + ) + exclude_files = {"codetables", "base"} + module_names = get_module_names(builtin_path, exclude_files) + import_builtins(module_names, mathics3_builtins_modules) + + # Get import modules in subdirectories of this directory of Python + # modules that contain Mathics3 Builtin class definitions. + + # The files_io module handles local file access, reading and writing.. + # In some sandboxed settings, such as running Mathics from as a remote + # server, we disallow local file access. + disable_file_module_names = set() if ENABLE_FILES_MODULE else {"files_io"} + + subdirectories = next(os.walk(builtin_path))[1] + import_builtin_subdirectories( + subdirectories, disable_file_module_names, mathics3_builtins_modules + ) + + add_builtins_from_builtin_modules(mathics3_builtins_modules) + initialize_display_operators_set() + + # TODO: When we drop Python 3.7, # module_names can be a List[Literal] def import_builtins( - module_names: List[str], modules: list, submodule_name: Optional[str] = None + module_names: List[str], + modules: List[ModuleType], + submodule_name: Optional[str] = None, ): """ Imports the list of Mathics3 Built-in modules so that inside @@ -174,7 +218,7 @@ def initialize_display_operators_set(): display_operators_set.add(operator) -def name_is_builtin_symbol(module, name: str) -> Optional[type]: +def name_is_builtin_symbol(module: ModuleType, name: str) -> Optional[type]: """ Checks if ``name`` should be added to definitions, and return its associated Builtin class. diff --git a/mathics/doc/common_doc.py b/mathics/doc/common_doc.py index f6c1a169e..617c43e27 100644 --- a/mathics/doc/common_doc.py +++ b/mathics/doc/common_doc.py @@ -31,10 +31,13 @@ from types import ModuleType from typing import Callable -from mathics import builtin, settings +from mathics import settings from mathics.builtin.base import check_requires_list from mathics.core.evaluation import Message, Print -from mathics.core.load_builtin import builtins_by_module as global_builtins_by_module +from mathics.core.load_builtin import ( + builtins_by_module as global_builtins_by_module, + mathics3_builtins_modules, +) from mathics.core.util import IS_PYPY from mathics.doc.utils import slugify from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules @@ -111,7 +114,8 @@ SUBSECTION_RE = re.compile('(?s)') TESTCASE_RE = re.compile( - r"""(?mx)^ # re.MULTILINE (multi-line match) and re.VERBOSE (readable regular expressions + r"""(?mx)^ # re.MULTILINE (multi-line match) + # and re.VERBOSE (readable regular expressions ((?:.|\n)*?) ^\s+([>#SX])>[ ](.*) # test-code indicator ((?:\n\s*(?:[:|=.][ ]|\.).*)*) # test-code results""" @@ -122,12 +126,6 @@ test_result_map = {} -def _replace_all(text, pairs): - for i, j in pairs: - text = text.replace(i, j) - return text - - def get_module_doc(module: ModuleType) -> tuple: doc = module.__doc__ if doc is not None: @@ -635,7 +633,7 @@ def gather_doctest_data(self): for title, modules, builtins_by_module, start in [ ( "Reference of Built-in Symbols", - builtin.modules, + mathics3_builtins_modules, global_builtins_by_module, True, ) diff --git a/mathics/doc/latex_doc.py b/mathics/doc/latex_doc.py index d17f6f7cc..8dc6a53e0 100644 --- a/mathics/doc/latex_doc.py +++ b/mathics/doc/latex_doc.py @@ -35,7 +35,6 @@ DocText, Documentation, XMLDoc, - _replace_all, gather_tests, get_results_by_test, post_sub, @@ -104,7 +103,7 @@ def repl_python(match): text, post_substitutions = pre_sub(PYTHON_RE, text, repl_python) - text = _replace_all( + text = replace_all( text, [ ("\\", "\\\\"), @@ -120,7 +119,7 @@ def repl_python(match): def repl(match): text = match.group(1) if text: - text = _replace_all(text, [("\\'", "'"), ("^", "\\^")]) + text = replace_all(text, [("\\'", "'"), ("^", "\\^")]) escape_char = get_latex_escape_char(text) text = LATEX_RE.sub( lambda m: "%s%s\\codevar{\\textit{%s}}%s\\lstinline%s" @@ -163,7 +162,7 @@ def repl_list(match): text = LIST_RE.sub(repl_list, text) # FIXME: get this from MathicsScanner - text = _replace_all( + text = replace_all( text, [ ("$", r"\$"), @@ -313,7 +312,7 @@ def repl_subsection(match): def escape_latex_output(text) -> str: """Escape Mathics output""" - text = _replace_all( + text = replace_all( text, [ ("\\", "\\\\"), @@ -448,6 +447,12 @@ def repl_nonasy(match): return OUTSIDE_ASY_RE.sub(repl_nonasy, result) +def replace_all(text, pairs): + for i, j in pairs: + text = text.replace(i, j) + return text + + def strip_system_prefix(name): if name.startswith("System`"): stripped_name = name[len("System`") :] diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py index a3be9bef1..b8d6f4cd4 100644 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -24,7 +24,11 @@ from mathics import settings, version_string from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation, Output -from mathics.core.load_builtin import builtins_by_module, builtins_dict +from mathics.core.load_builtin import ( + builtins_by_module, + builtins_dict, + import_and_load_builtins, +) from mathics.core.parser import MathicsSingleLineFeeder from mathics.doc.common_doc import MathicsMainDocumentation from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule @@ -177,7 +181,6 @@ def test_tests( max_tests=MAX_TESTS, excludes=[], ): - # For consistency set the character encoding ASCII which is # the lowest common denominator available on all systems. mathics.settings.SYSTEM_CHARACTER_ENCODING = "ASCII" @@ -487,6 +490,8 @@ def main(): global definitions global logfile global check_partial_elapsed_time + + import_and_load_builtins() definitions = Definitions(add_builtin=True) parser = ArgumentParser(description="Mathics test suite.", add_help=False) diff --git a/mathics/main.py b/mathics/main.py index 3d220a8e3..51eb3efbd 100755 --- a/mathics/main.py +++ b/mathics/main.py @@ -24,6 +24,7 @@ from mathics.core.definitions import Definitions, Symbol, autoload_files from mathics.core.evaluation import Evaluation, Output from mathics.core.expression import Expression +from mathics.core.load_builtin import import_and_load_builtins from mathics.core.parser import MathicsFileLineFeeder, MathicsLineFeeder from mathics.core.read import channel_to_stream from mathics.core.rules import BuiltinRule @@ -31,6 +32,12 @@ from mathics.core.symbols import SymbolNull, strip_context from mathics.timing import show_lru_cache_statistics +# from mathics.timing import TimeitContextManager +# with TimeitContextManager("import_and_load_builtins()"): +# import_and_load_builtins() + +import_and_load_builtins() + def get_srcdir(): filename = osp.normcase(osp.dirname(osp.abspath(__file__))) diff --git a/mathics/version.py b/mathics/version.py index 11bfc1b45..7b0a25141 100644 --- a/mathics/version.py +++ b/mathics/version.py @@ -5,4 +5,4 @@ # well as importing into Python. That's why there is no # space around "=" below. # fmt: off -__version__="6.0.2dev0" # noqa +__version__="7.0.0dev0" # noqa diff --git a/test/consistency-and-style/test_duplicate_builtins.py b/test/consistency-and-style/test_duplicate_builtins.py index 2e2a0dec8..5ece63ccb 100644 --- a/test/consistency-and-style/test_duplicate_builtins.py +++ b/test/consistency-and-style/test_duplicate_builtins.py @@ -8,9 +8,8 @@ import pytest -from mathics.builtin import modules from mathics.builtin.base import Builtin -from mathics.core.load_builtin import name_is_builtin_symbol +from mathics.core.load_builtin import mathics3_builtins_modules, name_is_builtin_symbol @pytest.mark.skipif( @@ -19,7 +18,7 @@ def test_check_duplicated(): msg = "" builtins_by_name = {} - for module in modules: + for module in mathics3_builtins_modules: vars = dir(module) for name in vars: var = name_is_builtin_symbol(module, name) @@ -33,15 +32,18 @@ def test_check_duplicated(): """ assert ( builtins_by_name.get(name, None) is None - ), f"{name} defined in {module} already defined in {builtins_by_name[name]}." + ), f"{name} defined in {module} already defined in " + f{builtins_by_name[name]}." """ # if builtins_by_name.get(name, None) is not None: # print( - # f"\n{name} defined in {module} already defined in {builtins_by_name[name]}." + # (f"\n{name} defined in {module} already defined in + # f{builtins_by_name[name]}.") # ) # msg = ( # msg - # + f"\n{name} defined in {module} already defined in {builtins_by_name[name]}." + # + (f"\n{name} defined in {module} already defined in " + # {builtins_by_name[name]}.") # ) builtins_by_name[name] = module assert msg == "", msg diff --git a/test/helper.py b/test/helper.py index 025dd3ca4..fa7b87d47 100644 --- a/test/helper.py +++ b/test/helper.py @@ -2,8 +2,11 @@ import time from typing import Optional +from mathics.core.load_builtin import import_and_load_builtins from mathics.session import MathicsSession +import_and_load_builtins() + # Set up a Mathics session with definitions. # For consistency set the character encoding ASCII which is # the lowest common denominator available on all systems.