Accomodation for debugging Get, and parse (#1089)

* Access parse() from Get in a way that allows a debugger to trap the call.
* Change the way Get[..., Trace->True] so that a debugger can hook into this more naturally -- separate the line number from the text.

Some other small changes were made like putting the Number class in alphabetic order, or adding more annotations.

mathics/builtin/files_io/files.py

@@ -4,7 +4,6 @@
 File and Stream Operations
 """
 
-import builtins
 import io
 import os.path as osp
 import tempfile
@@ -322,20 +321,6 @@ def eval(self, path, evaluation: Evaluation, options: dict):
         return SymbolNull
 
 
-class Number_(Builtin):
-    """
-    <url>:WMA link:https://reference.wolfram.com/language/ref/Number.html</url>
-
-    <dl>
-    <dt>'Number'
-      <dd>is a data type for 'Read'.
-    </dl>
-    """
-
-    name = "Number"
-    summary_text = "exact or approximate number in Fortran‐like notation"
-
-
 class Get(PrefixOperator):
     r"""
     <url>:WMA link:https://reference.wolfram.com/language/ref/Get.html</url>
@@ -346,8 +331,11 @@ class Get(PrefixOperator):
 
       <dt>'Get[$name$, Trace->True]'
       <dd>Runs Get tracing each line before it is evaluated.
+
+     'Settings`$TraceGet' can be also used to trace lines on all 'Get[]' calls.
     </dl>
 
+
     S> filename = $TemporaryDirectory <> "/example_file";
     S> Put[x + y, filename]
     S> Get[filename]
@@ -373,23 +361,21 @@ class Get(PrefixOperator):
     def eval(self, path: String, evaluation: Evaluation, options: dict):
         "Get[path_String, OptionsPattern[Get]]"
 
-        trace_fn = None
+        # Make sure to pick up copy from module each time instead of using
+        # use "from ... import DEFAULT_TRACE_FN" which will not pick
+        # up run-time changes made to the module function.
+        trace_fn = mathics.eval.files_io.files.DEFAULT_TRACE_FN
+
         trace_get = evaluation.parse("Settings`$TraceGet")
         if (
             options["System`Trace"].to_python()
             or trace_get.evaluate(evaluation) is SymbolTrue
         ):
-            trace_fn = builtins.print
+            trace_fn = print_line_number_and_text
 
         # perform the actual evaluation
         return eval_Get(path.value, evaluation, trace_fn)
 
-    def eval_default(self, filename, evaluation: Evaluation):
-        "Get[filename_]"
-        expr = to_expression("Get", filename)
-        evaluation.message("General", "stream", filename)
-        return expr
-
 
 class InputFileName_(Predefined):
     """
@@ -438,6 +424,20 @@ class InputStream(Builtin):
     summary_text = "an input stream"
 
 
+class Number_(Builtin):
+    """
+    <url>:WMA link:https://reference.wolfram.com/language/ref/Number.html</url>
+
+    <dl>
+    <dt>'Number'
+      <dd>is a data type for 'Read'.
+    </dl>
+    """
+
+    name = "Number"
+    summary_text = "exact or approximate number in Fortran‐like notation"
+
+
 class OpenRead(_OpenAction):
     """
     <url>

mathics/core/rules.py

@@ -198,7 +198,7 @@ class Rule(BaseRule):
     pattern and a replacement term and doesn't involve function
     application.
 
-    In contrast to BuiltinRule[], rule application cannot force
+    In contrast to FunctionApplyRule[], rule application cannot force
     a reevaluation of the expression when the rewrite/apply/eval step
     finishes.
 

mathics/core/streams.py

@@ -8,15 +8,20 @@
 import sys
 import tempfile
 from io import open as io_open
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
 
 import requests
 
 from mathics.core.util import canonic_filename
 from mathics.settings import ROOT_DIR
 
 HOME_DIR = osp.expanduser("~")
-PATH_VAR = [".", HOME_DIR, osp.join(ROOT_DIR, "data"), osp.join(ROOT_DIR, "packages")]
+PATH_VAR: List[str] = [
+    ".",
+    HOME_DIR,
+    osp.join(ROOT_DIR, "data"),
+    osp.join(ROOT_DIR, "packages"),
+]
 
 
 def create_temporary_file(prefix="Mathics3-", suffix=None, delete=True):

mathics/eval/files_io/files.py

@@ -3,12 +3,14 @@
 File related evaluation functions.
 """
 
-from typing import Callable, Optional
+from typing import Callable, Literal, Optional
 
 from mathics_scanner import TranslateError
 from mathics_scanner.errors import IncompleteSyntaxError, InvalidSyntaxError
 
 import mathics
+import mathics.core.parser
+import mathics.core.streams
 from mathics.core.builtin import MessageException
 from mathics.core.convert.expression import to_expression, to_mathics_list
 from mathics.core.convert.python import from_python
@@ -40,6 +42,15 @@
 #        $InputFilename
 INPUT_VAR: str = ""
 
+DEFAULT_TRACE_FN: Literal[None] = None
+
+
+def print_line_number_and_text(line_number: int, text: str):
+    """Prints a line number an text on that line with it.
+    This is used as the default trace function in Get[]
+    """
+    print(f"%5d: {text}" % line_number, end="")
+
 
 def set_input_var(input_string: str):
     """
@@ -49,7 +60,9 @@ def set_input_var(input_string: str):
     INPUT_VAR = canonic_filename(input_string)
 
 
-def eval_Get(path: str, evaluation: Evaluation, trace_fn: Optional[Callable]):
+def eval_Get(
+    path: str, evaluation: Evaluation, trace_fn: Optional[Callable] = DEFAULT_TRACE_FN
+):
     """
     Reads a file and evaluates each expression, returning only the last one.
     """
@@ -66,21 +79,23 @@ def eval_Get(path: str, evaluation: Evaluation, trace_fn: Optional[Callable]):
     outer_input_var = INPUT_VAR
     outer_inputfile = definitions.get_inputfile()
 
-    # set new input paths
+    # Set a new input path.
     INPUT_VAR = path
     definitions.set_inputfile(INPUT_VAR)
 
     mathics.core.streams.PATH_VAR = SymbolPath.evaluate(evaluation).to_python(
         string_quotes=False
     )
     if trace_fn is not None:
-        trace_fn(path)
+        trace_fn(0, path + "\n")
     try:
         with MathicsOpen(path, "r") as f:
             feeder = MathicsFileLineFeeder(f, trace_fn)
             while not feeder.empty():
                 try:
-                    query = parse(definitions, feeder)
+                    # Note: we use mathics.core.parser.parse
+                    # so that tracing/debugging can intercept parse()
+                    query = mathics.core.parser.parse(definitions, feeder)
                 except TranslateError:
                     return SymbolNull
                 finally:
@@ -95,7 +110,8 @@ def eval_Get(path: str, evaluation: Evaluation, trace_fn: Optional[Callable]):
         e.message(evaluation)
         return SymbolFailed
     finally:
-        # Always restore input paths of calling context.
+        # Whether we had an exception or not, restore the input path
+        # and the state of definitions prior to calling Get.
         INPUT_VAR = outer_input_var
         definitions.set_inputfile(outer_inputfile)
     return result