Merge pull request #875 from Mathics3/load-module-control

Move more builtin module loading code out of `mathics/builtin/__init__.py`

mathics/builtin/__init__.py

@@ -16,106 +16,37 @@
 among other things.
 """
 
-import glob
-import importlib
+import os
 import os.path as osp
-import pkgutil
-import re
 
-from mathics.builtin.base import Builtin
 from mathics.core.load_builtin import (
-    add_builtins,
+    add_builtins_from_builtin_modules,
+    get_module_names,
+    import_builtin_subdirectories,
     import_builtins,
-    name_is_builtin_symbol,
+    initialize_display_operators_set,
 )
 from mathics.settings import ENABLE_FILES_MODULE
 
-# Get a list of files in this directory. We'll exclude from the start
-# files with leading characters we don't want like __init__ with its leading underscore.
-__py_files__ = [
-    osp.basename(f[0:-3])
-    for f in glob.glob(osp.join(osp.dirname(__file__), "[a-z]*.py"))
-]
+# Get import modules in this directory of Python modules that contain
+# Mathics3 Builtin class definitions.
 
-# FIXME: redo using importlib since that is probably less fragile.
+builtin_path = osp.dirname(__file__)
 exclude_files = {"codetables", "base"}
-module_names = [
-    f for f in __py_files__ if re.match(r"^[a-z\d]+$", f) if f not in exclude_files
-]
-
+module_names = get_module_names(builtin_path, exclude_files)
 modules = []
-import_builtins(modules, module_names)
-
-_builtins_list = []
-builtins_by_module = {}
-
-disable_file_module_names = [] if ENABLE_FILES_MODULE else ["files_io"]
-
-for subdir in (
-    "arithfns",
-    "assignments",
-    "atomic",
-    "binary",
-    "box",
-    "colors",
-    "directories",
-    "distance",
-    "drawing",
-    "exp_structure",
-    "fileformats",
-    "files_io",
-    "file_operations",
-    "forms",
-    "functional",
-    "image",
-    "intfns",
-    "list",
-    "matrices",
-    "numbers",
-    "quantum_mechanics",
-    "specialfns",
-    "statistics",
-    "string",
-    "testing_expressions",
-    "vectors",
-):
-    import_name = f"{__name__}.{subdir}"
-
-    if subdir in disable_file_module_names:
-        continue
-
-    builtin_module = importlib.import_module(import_name)
-    submodule_names = [
-        modname for _, modname, _ in pkgutil.iter_modules(builtin_module.__path__)
-    ]
-    # print("XXX3", submodule_names)
-    import_builtins(modules, submodule_names, subdir)
-
-for module in modules:
-    builtins_by_module[module.__name__] = []
-    module_vars = dir(module)
-
-    for name in module_vars:
-        builtin_class = name_is_builtin_symbol(module, name)
-        if builtin_class is not None:
-            instance = builtin_class(expression=False)
-
-            if isinstance(instance, Builtin):
-                # This set the default context for symbols in mathics.builtins
-                if not type(instance).context:
-                    type(instance).context = "System`"
-                _builtins_list.append((instance.get_name(), instance))
-                builtins_by_module[module.__name__].append(instance)
+import_builtins(module_names, modules)
 
+# Get import modules in subdirectories of this directory of Python
+# modules that contain Mathics3 Builtin class definitions.
 
-new_builtins = _builtins_list
+# The files_io module handles local file access, reading and writing..
+# In some sandboxed settings, such as running Mathics from as a remote
+# server, we disallow local file access.
+disable_file_module_names = set() if ENABLE_FILES_MODULE else {"files_io"}
 
-add_builtins(new_builtins)
+subdirectories = next(os.walk(builtin_path))[1]
+import_builtin_subdirectories(subdirectories, disable_file_module_names, modules)
 
-display_operators_set = set()
-for modname, builtins in builtins_by_module.items():
-    for builtin in builtins:
-        # name = builtin.get_name()
-        operator = builtin.get_operator_display()
-        if operator is not None:
-            display_operators_set.add(operator)
+add_builtins_from_builtin_modules(modules)
+initialize_display_operators_set()

mathics/builtin/atomic/symbols.py

@@ -8,6 +8,8 @@
 
 import re
 
+from mathics_scanner import is_symbol_name
+
 from mathics.builtin.base import Builtin, PrefixOperator, Test
 from mathics.core.assignment import get_symbol_values
 from mathics.core.atoms import String
@@ -715,8 +717,6 @@ class Symbol_(Builtin):
     def eval(self, string, evaluation):
         "Symbol[string_String]"
 
-        from mathics.core.parser import is_symbol_name
-
         text = string.value
         if is_symbol_name(text):
             return Symbol(evaluation.definitions.lookup_name(string.value))

mathics/builtin/scoping.py

@@ -3,6 +3,7 @@
 Scoping Constructs
 """
 
+from mathics_scanner import is_symbol_name
 
 from mathics.builtin.base import Builtin, Predefined
 from mathics.core.assignment import get_symbol_list
@@ -618,8 +619,6 @@ def eval(self, evaluation: Evaluation):
     def eval_symbol(self, vars, attributes, evaluation: Evaluation):
         "Unique[vars_, attributes___]"
 
-        from mathics.core.parser import is_symbol_name
-
         attributes = attributes.get_sequence()
         if len(attributes) > 1:
             evaluation.message("Unique", "argrx", Integer(len(attributes) + 1))

mathics/core/load_builtin.py

@@ -3,19 +3,23 @@
 Code around loading Mathics3 Builtin Functions and Variables.
 
 This code loads the top-level definition of a Mathics3
-Builtin, and attributes that have not been segregated elsewhere such
-as has been done for one of the other modules listed above.
+Builtin.
 """
 
 import importlib
 import inspect
-from typing import Optional
+import os.path as osp
+import pkgutil
+from glob import glob
+from typing import List, Optional
 
 from mathics.core.pattern import pattern_objects
 from mathics.core.symbols import Symbol
 from mathics.eval.makeboxes import builtins_precedence
 
 _builtins = {}
+builtins_by_module = {}
+display_operators_set = set()
 
 # The fact that are importing inside here, suggests add_builtins
 # should get moved elsewhere.
@@ -46,6 +50,31 @@ def add_builtins(new_builtins):
     _builtins.update(dict(new_builtins))
 
 
+def add_builtins_from_builtin_modules(modules):
+    # This can be put at the top after mathics.builtin.__init__
+    # cleanup is done.
+    from mathics.builtin.base import Builtin
+
+    builtins_list = []
+    for module in modules:
+        builtins_by_module[module.__name__] = []
+        module_vars = dir(module)
+
+        for name in module_vars:
+            builtin_class = name_is_builtin_symbol(module, name)
+            if builtin_class is not None:
+                instance = builtin_class(expression=False)
+
+                if isinstance(instance, Builtin):
+                    # This set the default context for symbols in mathics.builtins
+                    if not type(instance).context:
+                        type(instance).context = "System`"
+                    builtins_list.append((instance.get_name(), instance))
+                    builtins_by_module[module.__name__].append(instance)
+    add_builtins(builtins_list)
+    return builtins_by_module
+
+
 def builtins_dict(builtins_by_module):
     return {
         builtin.get_name(): builtin
@@ -74,9 +103,18 @@ def definition_contribute(definitions):
             definitions.builtin[op] = Definition(name=op)
 
 
+def get_module_names(builtin_path: str, exclude_files: set) -> list:
+    py_files = [
+        osp.basename(f[0:-3]) for f in glob(osp.join(builtin_path, "[a-z]*.py"))
+    ]
+    return [f for f in py_files if f not in exclude_files]
+
+
 # TODO: When we drop Python 3.7,
 # module_names can be a List[Literal]
-def import_builtins(modules: list, module_names: list, submodule_name=None) -> None:
+def import_builtins(
+    module_names: List[str], modules: list, submodule_name: Optional[str] = None
+):
     """
     Imports the list of Mathics3 Built-in modules so that inside
     Mathics3 Builtin Functions, like Plus[], List[] are defined.
@@ -106,6 +144,36 @@ def import_module(module_name: str, import_name: str):
         import_module(module_name, import_name)
 
 
+def import_builtin_subdirectories(
+    subdirectories: List[str], disable_file_module_names: set, modules
+):
+    """
+    Runs import_builtisn on the each subdirectory in ``subdirectories`` that inside
+    Mathics3 Builtin Functions which are inside mathics.builtins.xxx are defined.
+    """
+    for subdir in subdirectories:
+        if subdir in disable_file_module_names:
+            continue
+
+        import_name = f"mathics.builtin.{subdir}"
+
+        builtin_module = importlib.import_module(import_name)
+        submodule_names = [
+            modname for _, modname, _ in pkgutil.iter_modules(builtin_module.__path__)
+        ]
+        # print("XXX3", submodule_names)
+        import_builtins(submodule_names, modules, subdir)
+
+
+def initialize_display_operators_set():
+    for _, builtins in builtins_by_module.items():
+        for builtin in builtins:
+            # name = builtin.get_name()
+            operator = builtin.get_operator_display()
+            if operator is not None:
+                display_operators_set.add(operator)
+
+
 def name_is_builtin_symbol(module, name: str) -> Optional[type]:
     """
     Checks if ``name`` should be added to definitions, and return

mathics/doc/common_doc.py

@@ -34,6 +34,7 @@
 from mathics import builtin, settings
 from mathics.builtin.base import check_requires_list
 from mathics.core.evaluation import Message, Print
+from mathics.core.load_builtin import builtins_by_module as global_builtins_by_module
 from mathics.core.util import IS_PYPY
 from mathics.doc.utils import slugify
 from mathics.eval.pymathics import pymathics_builtins_by_module, pymathics_modules
@@ -621,7 +622,7 @@ def gather_doctest_data(self):
             (
                 "Reference of Built-in Symbols",
                 builtin.modules,
-                builtin.builtins_by_module,
+                global_builtins_by_module,
                 True,
             )
         ]:

mathics/docpipeline.py

@@ -22,10 +22,9 @@
 import mathics
 import mathics.settings
 from mathics import settings, version_string
-from mathics.builtin import builtins_by_module
 from mathics.core.definitions import Definitions
 from mathics.core.evaluation import Evaluation, Output
-from mathics.core.load_builtin import builtins_dict
+from mathics.core.load_builtin import builtins_by_module, builtins_dict
 from mathics.core.parser import MathicsSingleLineFeeder
 from mathics.doc.common_doc import MathicsMainDocumentation
 from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule

mathics/eval/pymathics.py

@@ -6,10 +6,9 @@
 import inspect
 import sys
 
-from mathics.builtin import builtins_by_module
 from mathics.builtin.base import Builtin
 from mathics.core.definitions import Definitions
-from mathics.core.load_builtin import name_is_builtin_symbol
+from mathics.core.load_builtin import builtins_by_module, name_is_builtin_symbol
 
 # The below set and dictionary are used in document generation
 # for Pymathics modules.

mathics/format/mathml.py

@@ -8,6 +8,8 @@
 import base64
 import html
 
+from mathics_scanner import is_symbol_name
+
 from mathics.builtin.box.graphics import GraphicsBox
 from mathics.builtin.box.graphics3d import Graphics3DBox
 from mathics.builtin.box.layout import (
@@ -27,7 +29,7 @@
     add_conversion_fn,
     lookup_method as lookup_conversion_method,
 )
-from mathics.core.parser import is_symbol_name
+from mathics.core.load_builtin import display_operators_set as operators
 from mathics.core.symbols import SymbolTrue
 
 
@@ -59,8 +61,6 @@ def encode_mathml(text: str) -> str:
 
 
 def string(self, **options) -> str:
-    from mathics.builtin import display_operators_set as operators
-
     text = self.value
 
     number_as_text = options.get("number_as_text", None)

test/consistency-and-style/test_duplicate_builtins.py

@@ -8,8 +8,9 @@
 
 import pytest
 
-from mathics.builtin import modules, name_is_builtin_symbol
+from mathics.builtin import modules
 from mathics.builtin.base import Builtin
+from mathics.core.load_builtin import name_is_builtin_symbol
 
 
 @pytest.mark.skipif(

test/consistency-and-style/test_summary_text.py

@@ -7,8 +7,8 @@
 import pytest
 
 from mathics import __file__ as mathics_initfile_path
-from mathics.builtin import name_is_builtin_symbol
 from mathics.builtin.base import Builtin
+from mathics.core.load_builtin import name_is_builtin_symbol
 from mathics.doc.common_doc import skip_doc
 
 # Get file system path name for mathics.builtin