Table driven operator precedence (#1075)

Prefer operator precedence from mathics-scanner tables.

---------

Co-authored-by: Juan Mauricio Matera <[email protected]>

.github/workflows/osx.yml

@@ -29,8 +29,8 @@ jobs:
     - name: Install Mathics3 with full Python dependencies
       run: |
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        python -m pip install Mathics-Scanner
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        # python -m pip install Mathics-Scanner[full]
         make develop-full
     - name: Test Mathics3
       run: |

.github/workflows/ubuntu-cython.yml

@@ -25,8 +25,8 @@ jobs:
         sudo apt-get update -qq && sudo apt-get install -qq liblapack-dev llvm-dev tesseract-ocr
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        python -m pip install Mathics-Scanner
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        # python -m pip install Mathics-Scanner[full]
     - name: Install Mathics with full dependencies
       run: |
         make develop-full-cython

.github/workflows/ubuntu.yml

@@ -25,8 +25,8 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        python -m pip install Mathics-Scanner
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        # python -m pip install Mathics-Scanner[full]
         make develop-full
     - name: Test Mathics
       run: |

.github/workflows/windows.yml

@@ -34,8 +34,8 @@ jobs:
     - name: Install Mathics3 with Python dependencies
       run: |
         # We can comment out after next Mathics-Scanner release
-        # python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
-        python -m pip install Mathics-Scanner
+        python -m pip install -e git+https://github.com/Mathics3/mathics-scanner#egg=Mathics-Scanner[full]
+        # python -m pip install Mathics-Scanner[full]
         make develop-full
     - name: Test Mathics3
       # Limit pip install to a basic install *without* full dependencies.

Makefile

@@ -61,17 +61,17 @@ build:
 # because pip install doesn't handle
 # INSTALL_REQUIRES properly
 #: Set up to run from the source tree
-develop:  mathics/data/op-tables.json
+develop:  mathics/data/op-tables.json mathics/data/operators.json
 	$(PIP) install -e .[dev]
 
 # See note above on ./setup.py
 #: Set up to run from the source tree with full dependencies
-develop-full:  mathics/data/op-tables.json
+develop-full:  mathics/data/op-tables.json mathics/data/operators.json
 	$(PIP) install -e .[dev,full]
 
 # See note above on ./setup.py
 #: Set up to run from the source tree with full dependencies and Cython
-develop-full-cython: mathics/data/op-tables.json
+develop-full-cython: mathics/data/op-tables.json mathics/data/operators.json
 	$(PIP) install -e .[dev,full,cython]
 
 
@@ -141,10 +141,11 @@ doctest:
 latexdoc texdoc doc:
 	(cd mathics/doc/latex && $(MAKE) doc)
 
-#: Build JSON ASCII to unicode opcode tables
-mathics/data/op-tables.json:
+#: Build JSON ASCII to unicode opcode table and operator table
+mathics/data/op-tables.json mathics/data/operators.json:
 	$(BASH) ./admin-tools/make-op-tables.sh
 
+
 #: Remove ChangeLog
 rmChangeLog:
 	$(RM) ChangeLog || true

admin-tools/make-op-tables.sh

@@ -5,10 +5,11 @@ mydir=$(dirname $bs)
 PYTHON=${PYTHON:-python}
 
 cd $mydir/../mathics/data
-mathics-generate-json-table \
+mathics3-generate-json-table \
     --field=ascii-operator-to-symbol \
     --field=ascii-operator-to-unicode \
     --field=ascii-operator-to-wl-unicode \
     --field=operator-to-ascii \
     --field=operator-to-unicode \
     -o op-tables.json
+mathics3-generate-operator-json-table -o operator-tables.json

mathics/core/builtin.py

@@ -7,12 +7,14 @@
 """
 
 import importlib
+import os.path as osp
 import re
 from functools import lru_cache, total_ordering
 from itertools import chain
 from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union, cast
 
 import mpmath
+import pkg_resources
 import sympy
 
 from mathics.core.atoms import (
@@ -67,6 +69,21 @@
 from mathics.eval.numerify import numerify
 from mathics.eval.scoping import dynamic_scoping
 
+try:
+    import ujson
+except ImportError:
+    import json as ujson
+
+ROOT_DIR = pkg_resources.resource_filename("mathics", "")
+
+# Load the conversion tables from disk
+characters_path = osp.join(ROOT_DIR, "data", "operator-tables.json")
+assert osp.exists(
+    characters_path
+), f"Operator precedence tables are missing; expected to be in {characters_path}"
+with open(characters_path, "r") as f:
+    operator_data = ujson.load(f)
+
 
 class Builtin:
     """
@@ -982,16 +999,16 @@ def __init__(self, format_function, *args, **kwargs):
         super().__init__(*args, **kwargs)
         name = self.get_name()
         if self.needs_verbatim:
-            name = "Verbatim[%s]" % name
+            name = f"Verbatim[{name}"
         if self.default_formats:
-            op_pattern = "%s[item_]" % name
+            op_pattern = f"{name}[item_]"
             if op_pattern not in self.formats:
                 operator = self.get_operator_display()
                 if operator is not None:
                     form = '%s[{HoldForm[item]},"%s",%d]' % (
                         format_function,
                         operator,
-                        self.precedence,
+                        operator_data["operator-precedence"].get(name, self.precedence),
                     )
                     self.formats[op_pattern] = form
 
@@ -1032,7 +1049,7 @@ def __init__(self, *args, **kwargs):
             formatted = "MakeBoxes[Infix[{%s}, %s, %d,%s], form]" % (
                 replace_items,
                 operator,
-                self.precedence,
+                operator_data["operator-precedence"].get(name, self.precedence),
                 self.grouping,
             )
             default_rules = {

mathics/data/.gitignore

@@ -1,3 +1,4 @@
 /doc_latex_data.pcl
 /doctest_latex_data.pcl
 /op-tables.json
+/operator-tables.json

setup.py

@@ -102,14 +102,18 @@ class build_py(setuptools_build_py):
     def run(self):
         if not os.path.exists("mathics/data/op-tables.json"):
             os.system(
-                "mathics-generate-json-table"
+                "mathics3-generate-json-table"
                 " --field=ascii-operator-to-symbol"
                 " --field=ascii-operator-to-unicode"
                 " --field=ascii-operator-to-wl-unicode"
                 " --field=operator-to-ascii"
                 " --field=operator-to-unicode"
                 " -o mathics/data/op-tables.json"
             )
+        if not os.path.exists("mathics/data/operator-tables.json"):
+            os.system(
+                "mathics3-generate-operator-json-table" " -o operator-tables.json"
+            )
         self.distribution.package_data["mathics"].append("data/op-tables.json")
         setuptools_build_py.run(self)