From cf096540c27b24fb37037932002ae7350cd53ad7 Mon Sep 17 00:00:00 2001 From: Mauricio Matera Date: Fri, 9 Feb 2024 15:21:43 -0300 Subject: [PATCH 1/6] bark when a rule in Builtin.rules cannot be loaded --- mathics/core/definitions.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mathics/core/definitions.py b/mathics/core/definitions.py index 1d1cdb26c..a6c3d33b4 100644 --- a/mathics/core/definitions.py +++ b/mathics/core/definitions.py @@ -815,7 +815,8 @@ def __init__( self.defaultvalues = defaultvalues self.builtin = builtin for rule in rules: - self.add_rule(rule) + if not self.add_rule(rule): + print(f"{rule.pattern.expr} could not be associated to {self.name}") def get_values_list(self, pos): assert pos.isalpha() From 4d25cbba5031c12ed3992e8973b9815603dbd1f4 Mon Sep 17 00:00:00 2001 From: mmatera Date: Sat, 10 Feb 2024 15:27:14 -0300 Subject: [PATCH 2/6] another round --- mathics/builtin/datentime.py | 6 ------ mathics/builtin/files_io/files.py | 4 +++- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/mathics/builtin/datentime.py b/mathics/builtin/datentime.py index 0f31fef91..9c9342cf6 100644 --- a/mathics/builtin/datentime.py +++ b/mathics/builtin/datentime.py @@ -1200,12 +1200,6 @@ class TimeZone(Predefined): summary_text = "gets the default time zone" - def eval(self, lhs, rhs, evaluation): - "lhs_ = rhs_" - - self.assign(lhs, rhs, evaluation) - return rhs - def evaluate(self, evaluation) -> Real: return self.value diff --git a/mathics/builtin/files_io/files.py b/mathics/builtin/files_io/files.py index a36f1a3b1..4d85758cc 100644 --- a/mathics/builtin/files_io/files.py +++ b/mathics/builtin/files_io/files.py @@ -1075,7 +1075,7 @@ def eval(self, channel, types, evaluation: Evaluation, options: dict): return from_python(result) def eval_nostream(self, arg1, arg2, evaluation): - "Read[arg1_, arg2_]" + "%(name)s[arg1_, arg2_]" evaluation.message("General", "stream", arg1) return @@ -1428,6 +1428,8 @@ class Find(Read): = ... """ + rules = {} + options = { "AnchoredSearch": "False", "IgnoreCase": "False", From 3e6aa72175f052cdbbb8041f71e9bf42e90a21f4 Mon Sep 17 00:00:00 2001 From: mmatera Date: Sat, 10 Feb 2024 21:18:55 -0300 Subject: [PATCH 3/6] more fixes. Improving get_tag_name and adding tests --- mathics/builtin/box/layout.py | 4 +- mathics/builtin/list/rearrange.py | 4 +- mathics/core/definitions.py | 125 ++++++++++++++++++++++++++---- test/core/test_definitions.py | 59 ++++++++++++++ 4 files changed, 174 insertions(+), 18 deletions(-) create mode 100644 test/core/test_definitions.py diff --git a/mathics/builtin/box/layout.py b/mathics/builtin/box/layout.py index 517195340..5dc932938 100644 --- a/mathics/builtin/box/layout.py +++ b/mathics/builtin/box/layout.py @@ -203,11 +203,11 @@ class InterpretationBox(BoxExpression): summary_text = "box associated to an input expression" def eval_to_expression(boxexpr, form, evaluation): - """ToExpression[boxexpr_IntepretationBox, form___]""" + """ToExpression[boxexpr_InterpretationBox, form___]""" return boxexpr.elements[1] def eval_display(boxexpr, evaluation): - """DisplayForm[boxexpr_IntepretationBox]""" + """DisplayForm[boxexpr_InterpretationBox]""" return boxexpr.elements[0] diff --git a/mathics/builtin/list/rearrange.py b/mathics/builtin/list/rearrange.py index d69a88c88..8b280ec8b 100644 --- a/mathics/builtin/list/rearrange.py +++ b/mathics/builtin/list/rearrange.py @@ -963,7 +963,7 @@ class Partition(Builtin):
'Partition[$list$, $n$]'
partitions $list$ into sublists of length $n$. -
'Parition[$list$, $n$, $d$]' +
'Partition[$list$, $n$, $d$]'
partitions $list$ into sublists of length $n$ which overlap $d$ \ indices. @@ -982,7 +982,7 @@ class Partition(Builtin): """ summary_text = "partition a list into sublists of a given length" rules = { - "Parition[list_, n_, d_, k]": "Partition[list, n, d, {k, k}]", + "Partition[list_, n_, d_, k]": "Partition[list, n, d, {k, k}]", } def _partition(self, expr, n, d, evaluation: Evaluation): diff --git a/mathics/core/definitions.py b/mathics/core/definitions.py index a6c3d33b4..9f4fa7ca9 100644 --- a/mathics/core/definitions.py +++ b/mathics/core/definitions.py @@ -722,25 +722,122 @@ def get_history_length(self): def get_tag_position(pattern, name) -> Optional[str]: + """ + Determine the position of a pattern in + the definition of the symbol ``name`` + """ + blanks = ( + "System`Blank", + "System`BlankSequence", + "System`BlankNullSequence", + ) + + def strip_pattern_name_and_condition(pat): + """ + In ``Pattern[name_, pattern_]`` and + ``Condition[pattern_, cond_]`` + the tag is determined by pat. + This function strips it to ensure that + ``pat`` does not have that form. + """ + if pat.get_head_name() == "System`Condition": + if len(pat.elements) > 1: + return strip_pattern_name_and_condition(pat.elements[0]) + if pat.get_head_name() == "System`Pattern": + if len(pat.elements) == 2: + return strip_pattern_name_and_condition(pat.elements[1]) + return pat + + def check_is_subvalue(pattern_sv, name_sv): + """Determines if ``pattern`` is a subvalue of ``name``""" + if name_sv == pattern_sv.get_lookup_name(): + return True + + # Try again after strip Pattern and Condition wrappers: + head = strip_pattern_name_and_condition(pattern_sv.get_head()) + head_name = head.get_lookup_name() + if name_sv == head_name: + return True + # The head is of the form ``_SymbolName|__SymbolName|___SymbolName`` + # If name matches with SymbolName, then is a subvalue: + if head_name in blanks: + if isinstance(head, Symbol): + return False + sub_elements = head.elements + if len(sub_elements) == 1: + head_name = head.elements[0].get_name() + if head_name == name_sv: + return True + return False + + # If pattern is a Symbol, and coincides with + # name, it is an ownvalue: + if pattern.get_name() == name: return "own" - elif isinstance(pattern, Atom): + # If pattern is an ``Atom``, does not have + # a position + if isinstance(pattern, Atom): return None - else: - head_name = pattern.get_head_name() - if head_name == name: - return "down" - elif head_name == "System`N" and len(pattern.elements) == 2: + + # The pattern is an Expression. + head_name = pattern.get_head_name() + # If the name is the head name, is a downvalue: + if head_name == name: + return "down" + + # Handle special cases + if head_name == "System`N": + if len(pattern.elements) == 2: return "n" - elif head_name == "System`Condition" and len(pattern.elements) > 0: - return get_tag_position(pattern.elements[0], name) - elif pattern.get_lookup_name() == name: - return "sub" - else: - for element in pattern.elements: - if element.get_lookup_name() == name: + + # The pattern has the form `_SymbolName | __SymbolName | ___SymbolName` + # Then it only can be a downvalue + if head_name in blanks: + elements = pattern.elements + if len(elements) == 1: + head_name = elements[0].get_name() + return "down" if head_name == name else None + + # TODO: Consider process format_values + + if head_name != "": + # Check + strip_pattern = strip_pattern_name_and_condition(pattern) + if strip_pattern is not pattern: + return get_tag_position(strip_pattern, name) + + # Check if ``pattern`` is a subvalue: + + # The head is not a symbol. pattern is a subvalue? + if check_is_subvalue(pattern, name): + return "sub" + + # If we are here, pattern is not an Ownvalue, DownValue, SubValue or NValue + # Let's check the elements for UpValues + for element in pattern.elements: + lookup_name = element.get_lookup_name() + if lookup_name == name: + return "up" + + # Strip Pattern and Condition wrappers and check again + if lookup_name in ( + "System`Condition", + "System`Pattern", + ): + element = strip_pattern_name_and_condition(element) + lookup_name = element.get_lookup_name() + if lookup_name == name: + return "up" + # Check if one of the elements is not a "Blank" + + if element.get_head_name() in blanks: + sub_elements = element.elements + if len(sub_elements) == 1: + if sub_elements[0].get_name() == name: return "up" - return None + # ``pattern`` does not have a tag position in the Definition + return None def insert_rule(values, rule) -> None: diff --git a/test/core/test_definitions.py b/test/core/test_definitions.py new file mode 100644 index 000000000..43eb160b0 --- /dev/null +++ b/test/core/test_definitions.py @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +""" +Tests functions in mathics.core.definition +""" + +import pytest + +from mathics.core.definitions import get_tag_position +from mathics.core.parser import parse_builtin_rule + + +@pytest.mark.parametrize( + ("pattern_str", "tag", "position"), + [ + # None + ("A", "B", None), + ("A_", "B", None), + ("A[c_]", "B", None), + ("A[3]", "B", None), + ("A[B][3]", "B", None), + ("A[s[x_]][y]", "s", None), + # Ownvalues + ("A", "A", "own"), + ("A/;A>0", "A", "own"), + ("s:(A/;A>0)", "A", "own"), + ("(s:A)/;A>0", "A", "own"), + ("s:A/;A>0", "A", "own"), + # Downvalues + ("_A", "A", "down"), + ("A[]", "A", "down"), + ("_A", "A", "down"), + ("A[p_, q]", "A", "down"), + ("s:A[p_, q]", "A", "down"), + ("A[p_, q]/;q>0", "A", "down"), + ("(s:A[p_, q])/;q>0", "A", "down"), + # NValues + ("N[A[x_], _]", "A", "n"), + ("N[A[x_], _]/; x>0", "A", "n"), + # Subvalues + ("_A[]", "A", "sub"), + ("A[x][t]", "A", "sub"), + ("(s:A[x])[t]", "A", "sub"), + ("(x_A/;u>0)[p]", "A", "sub"), + # Upvalues + ("S[x_, A]", "A", "up"), + ("S[x_, _A]", "A", "up"), + ("S[x_, s_A/;s>0]", "A", "up"), + ("S[x_, q:A]", "A", "up"), + ("S[x_, q:(A[t_]/;t>0)]", "A", "up"), + ("A[x_][s[y]]", "s", "up"), + ("DisplayForm[boxexpr_InterpretationBox]", "InterpretationBox", "up"), + ("ToExpression[boxexpr_InterpretationBox, form___]", "InterpretationBox", "up"), + # Just one argument, must be an upvalue + ("N[A[s_]]", "A", "up"), + ], +) +def test_get_tag_position(pattern_str, tag, position): + pattern = parse_builtin_rule(pattern_str) + assert get_tag_position(pattern, f"System`{tag}") == position From 07dbe3af882a369c069c85522ec68db6dbd6a69a Mon Sep 17 00:00:00 2001 From: mmatera Date: Sun, 11 Feb 2024 07:25:21 -0300 Subject: [PATCH 4/6] * Fix some typos in the Builtin rules * Improve ``mathics.core.definitions.get_tag_position`` to support more complex rules * Add a pytest to check different cases of ``get_tag_position`` * Restore ``mathics.docpipeline`` from master~3, because the current one is not properly working. --- mathics/docpipeline.py | 848 ++++++++++++----------------------------- 1 file changed, 250 insertions(+), 598 deletions(-) mode change 100755 => 100644 mathics/docpipeline.py diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py old mode 100755 new mode 100644 index c75b8c2cd..7c1e36586 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# FIXME: combine with same thing in Django +# FIXME: combine with same thing in Mathics core """ Does 2 things which can either be done independently or as a pipeline: @@ -17,31 +17,33 @@ import sys from argparse import ArgumentParser from datetime import datetime -from typing import Dict, Optional, Set, Tuple, Union +from typing import Dict, Optional import mathics +import mathics.settings from mathics import settings, version_string from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation, Output -from mathics.core.load_builtin import _builtins, import_and_load_builtins -from mathics.core.parser import MathicsSingleLineFeeder -from mathics.doc.common_doc import ( - DocGuideSection, - DocSection, - DocTest, - DocTests, - MathicsMainDocumentation, +from mathics.core.load_builtin import ( + builtins_by_module, + builtins_dict, + import_and_load_builtins, ) -from mathics.doc.utils import load_doctest_data, print_and_log +from mathics.core.parser import MathicsSingleLineFeeder +from mathics.doc.common_doc import MathicsMainDocumentation from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule from mathics.timing import show_lru_cache_statistics +builtins = builtins_dict(builtins_by_module) + class TestOutput(Output): def max_stored_size(self, _): return None +sep = "-" * 70 + "\n" + # Global variables definitions = None documentation = None @@ -49,22 +51,20 @@ def max_stored_size(self, _): logfile = None -# FIXME: After 3.8 is the minimum Python we can turn "str" into a Literal -SEP: str = "-" * 70 + "\n" -STARS: str = "*" * 10 - -DEFINITIONS = None -DOCUMENTATION = None -CHECK_PARTIAL_ELAPSED_TIME = False -LOGFILE = None +MAX_TESTS = 100000 # Number than the total number of tests -MAX_TESTS = 100000 # A number greater than the total number of tests. +def print_and_log(*args): + a = [a.decode("utf-8") if isinstance(a, bytes) else a for a in args] + string = "".join(a) + print(string) + if logfile: + logfile.write(string) -def doctest_compare(result: Optional[str], wanted: Optional[str]) -> bool: +def compare(result: Optional[str], wanted: Optional[str]) -> bool: """ - Performs a doctest comparison between ``result`` and ``wanted`` and returns + Performs test comparision betewen ``result`` and ``wanted`` and returns True if the test should be considered a success. """ if wanted in ("...", result): @@ -72,70 +72,57 @@ def doctest_compare(result: Optional[str], wanted: Optional[str]) -> bool: if result is None or wanted is None: return False - - result_list = result.splitlines() - wanted_list = wanted.splitlines() - if result_list == [] and wanted_list == ["#<--#"]: + result = result.splitlines() + wanted = wanted.splitlines() + if result == [] and wanted == ["#<--#"]: return True - if len(result_list) != len(wanted_list): + if len(result) != len(wanted): return False - for res, want in zip(result_list, wanted_list): - wanted_re = re.escape(want.strip()) + for r, w in zip(result, wanted): + wanted_re = re.escape(w.strip()) wanted_re = wanted_re.replace("\\.\\.\\.", ".*?") wanted_re = f"^{wanted_re}$" - if not re.match(wanted_re, res.strip()): + if not re.match(wanted_re, r.strip()): return False return True -def test_case( - test: DocTest, - index: int = 0, - subindex: int = 0, - quiet: bool = False, - section_name: str = "", - section_for_print="", - chapter_name: str = "", - part: str = "", -) -> bool: - """ - Run a single test cases ``test``. Return True if test succeeds and False if it - fails. ``index``gives the global test number count, while ``subindex`` counts - from the beginning of the section or subsection. +stars = "*" * 10 - The test results are assumed to be foramtted to ASCII text. - """ - global CHECK_PARTIAL_ELAPSED_TIME - test_str, wanted_out, wanted = test.test, test.outs, test.result +def test_case( + test, tests, index=0, subindex=0, quiet=False, section=None, format="text" +) -> bool: + global check_partial_elapsed_time + test, wanted_out, wanted = test.test, test.outs, test.result def fail(why): + part, chapter, section = tests.part, tests.chapter, tests.section print_and_log( - LOGFILE, - f"""{SEP}Test failed: in {part} / {chapter_name} / {section_name} + f"""{sep}Test failed: {section} in {part} / {chapter} {part} {why} """.encode( "utf-8" - ), + ) ) return False if not quiet: - if section_for_print: - print(f"{STARS} {section_for_print} {STARS}") - print(f"{index:4d} ({subindex:2d}): TEST {test_str}") + if section: + print(f"{stars} {tests.chapter} / {section} {stars}".encode("utf-8")) + print(f"{index:4d} ({subindex:2d}): TEST {test}".encode("utf-8")) - feeder = MathicsSingleLineFeeder(test_str, filename="") + feeder = MathicsSingleLineFeeder(test, "") evaluation = Evaluation( - DEFINITIONS, catch_interrupt=False, output=TestOutput(), format="text" + definitions, catch_interrupt=False, output=TestOutput(), format=format ) try: time_parsing = datetime.now() query = evaluation.parse_feeder(feeder) - if CHECK_PARTIAL_ELAPSED_TIME: + if check_partial_elapsed_time: print(" parsing took", datetime.now() - time_parsing) if query is None: # parsed expression is None @@ -143,25 +130,24 @@ def fail(why): out = evaluation.out else: result = evaluation.evaluate(query) - if CHECK_PARTIAL_ELAPSED_TIME: + if check_partial_elapsed_time: print(" evaluation took", datetime.now() - time_parsing) out = result.out result = result.result except Exception as exc: - fail(f"Exception {exc}") + fail("Exception %s" % exc) info = sys.exc_info() sys.excepthook(*info) return False time_comparing = datetime.now() - comparison_result = doctest_compare(result, wanted) + comparison_result = compare(result, wanted) - if CHECK_PARTIAL_ELAPSED_TIME: + if check_partial_elapsed_time: print(" comparison took ", datetime.now() - time_comparing) - if not comparison_result: - print("result != wanted") - fail_msg = f"Result: {result}\nWanted: {wanted}" + print("result =!=wanted") + fail_msg = "Result: %s\nWanted: %s" % (result, wanted) if out: fail_msg += "\nAdditional output:\n" fail_msg += "\n".join(str(o) for o in out) @@ -172,7 +158,7 @@ def fail(why): # If we have ... don't check pass elif len(out) != len(wanted_out): - # Mismatched number of output lines, and we don't have "..." + # Mismatched number of output lines and we don't have "..." output_ok = False else: # Need to check all output line by line @@ -180,7 +166,7 @@ def fail(why): if not got == wanted and wanted.text != "...": output_ok = False break - if CHECK_PARTIAL_ELAPSED_TIME: + if check_partial_elapsed_time: print(" comparing messages took ", datetime.now() - time_comparing) if not output_ok: return fail( @@ -190,28 +176,67 @@ def fail(why): return True -def create_output(tests, doctest_data, output_format="latex"): - if DEFINITIONS is None: - print_and_log(LOGFILE, "Definitions are not initialized.") - return +def test_tests( + tests, + index, + quiet=False, + stop_on_failure=False, + start_at=0, + max_tests=MAX_TESTS, + excludes=[], +): + # For consistency set the character encoding ASCII which is + # the lowest common denominator available on all systems. + mathics.settings.SYSTEM_CHARACTER_ENCODING = "ASCII" + + definitions.reset_user_definitions() + total = failed = skipped = 0 + failed_symbols = set() + section = tests.section + if section in excludes: + return total, failed, len(tests.tests), failed_symbols, index + count = 0 + for subindex, test in enumerate(tests.tests): + index += 1 + if test.ignore: + continue + if index < start_at: + skipped += 1 + continue + elif count >= max_tests: + break + + total += 1 + count += 1 + if not test_case(test, tests, index, subindex + 1, quiet, section): + failed += 1 + failed_symbols.add((tests.part, tests.chapter, tests.section)) + if stop_on_failure: + break + + section = None + return total, failed, skipped, failed_symbols, index - DEFINITIONS.reset_user_definitions() - for test in tests: + +# FIXME: move this to common routine +def create_output(tests, doctest_data, format="latex"): + definitions.reset_user_definitions() + for test in tests.tests: if test.private: continue key = test.key evaluation = Evaluation( - DEFINITIONS, format=output_format, catch_interrupt=True, output=TestOutput() + definitions, format=format, catch_interrupt=True, output=TestOutput() ) try: result = evaluation.parse_evaluate(test.test) - except Exception: # noqa + except: # noqa result = None if result is None: result = [] else: result_data = result.get_data() - result_data["form"] = output_format + result_data["form"] = format result = [result_data] doctest_data[key] = { @@ -220,472 +245,102 @@ def create_output(tests, doctest_data, output_format="latex"): } -def show_test_summary( - total: int, - failed: int, - entity_name: str, - entities_searched: str, - keep_going: bool, - generate_output: bool, - output_data, +def test_chapters( + chapters: set, + quiet=False, + stop_on_failure=False, + generate_output=False, + reload=False, + keep_going=False, ): - """ - Print and log test summary results. - - If ``generate_output`` is True, we will also generate output data - to ``output_data``. - """ - - print() - if total == 0: - print_and_log( - LOGFILE, f"No {entity_name} found with a name in: {entities_searched}." - ) - if "MATHICS_DEBUG_TEST_CREATE" not in os.environ: - print(f"Set environment MATHICS_DEBUG_TEST_CREATE to see {entity_name}.") - elif failed > 0: - print(SEP) - if not generate_output: - print_and_log( - LOGFILE, f"""{failed} test{'s' if failed != 1 else ''} failed.""" - ) - else: - print_and_log(LOGFILE, "All tests passed.") - - if generate_output and (failed == 0 or keep_going): - save_doctest_data(output_data) - return - - -def test_section_in_chapter( - section: Union[DocSection, DocGuideSection], - total: int, - failed: int, - quiet, - stop_on_failure, - prev_key: list, - include_sections: Optional[Set[str]] = None, - start_at: int = 0, - skipped: int = 0, - max_tests: int = MAX_TESTS, -) -> Tuple[int, int, list]: - """ - Runs a tests for section ``section`` under a chapter or guide section. - Note that both of these contain a collection of section tests underneath. - - ``total`` and ``failed`` give running tallies on the number of tests run and - the number of tests respectively. - - If ``quiet`` is True, the progress and results of the tests are shown. - If ``stop_on_failure`` is true then the remaining tests in a section are skipped when a test - fails. - """ - section_name = section.title - - # Start out assuming all subsections will be tested - include_subsections = None - - if include_sections is not None and section_name not in include_sections: - # use include_section to filter subsections - include_subsections = include_sections - - chapter = section.chapter - chapter_name = chapter.title - part_name = chapter.part.title + failed = 0 index = 0 - if len(section.subsections) > 0: - for subsection in section.subsections: - if ( - include_subsections is not None - and subsection.title not in include_subsections - ): - continue - - DEFINITIONS.reset_user_definitions() - for test in subsection.doc.get_tests(): - # Get key dropping off test index number - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - section_name_for_print = " / ".join(key) - if quiet: - # We don't print with stars inside in test_case(), so print here. - print(f"Testing section: {section_name_for_print}") - index = 0 - else: - # Null out section name, so that on the next iteration we do not print a section header - # in test_case(). - section_name_for_print = "" - - if isinstance(test, DocTests): - for doctest in test.tests: - index += 1 - total += 1 - if not test_case( - doctest, - total, - index, - quiet=quiet, - section_name=section_name, - section_for_print=section_name_for_print, - chapter_name=chapter_name, - part=part_name, - ): - failed += 1 - if stop_on_failure: - break - elif test.ignore: - continue - - else: - index += 1 - - if index < start_at: - skipped += 1 - continue - - total += 1 - if not test_case( - test, - total, - index, - quiet=quiet, - section_name=section_name, - section_for_print=section_name_for_print, - chapter_name=chapter_name, - part=part_name, - ): - failed += 1 - if stop_on_failure: - break - pass - pass - pass - pass - else: - if include_subsections is None or section.title in include_subsections: - DEFINITIONS.reset_user_definitions() - for test in section.doc.get_tests(): - # Get key dropping off test index number + chapter_names = ", ".join(chapters) + print(f"Testing chapter(s): {chapter_names}") + output_data = load_doctest_data() if reload else {} + prev_key = [] + for tests in documentation.get_tests(): + if tests.chapter in chapters: + for test in tests.tests: key = list(test.key)[1:-1] if prev_key != key: prev_key = key - section_name_for_print = " / ".join(key) - if quiet: - print(f"Testing section: {section_name_for_print}") + print(f'Testing section: {" / ".join(key)}') index = 0 - else: - # Null out section name, so that on the next iteration we do not print a section header. - section_name_for_print = "" - if test.ignore: continue - - else: - index += 1 - - if index < start_at: - skipped += 1 - continue - - total += 1 - if total >= max_tests: + index += 1 + if not test_case(test, tests, index, quiet=quiet): + failed += 1 + if stop_on_failure: break + if generate_output and failed == 0: + create_output(tests, output_data) - if not test_case( - test, - total, - index, - quiet=quiet, - section_name=section.title, - section_for_print=section_name_for_print, - chapter_name=chapter.title, - part=part_name, - ): - failed += 1 - if stop_on_failure: - break - pass - pass - - pass - return total, failed, prev_key - - -# When 3.8 is base, the below can be a Literal type. -INVALID_TEST_GROUP_SETUP = (None, None) - - -def validate_group_setup( - include_set: set, - entity_name: Optional[str], - reload: bool, -) -> tuple: - """ - Common things that need to be done before running a group of doctests. - """ - - if DOCUMENTATION is None: - print_and_log(LOGFILE, "Documentation is not initialized.") - return INVALID_TEST_GROUP_SETUP - - if entity_name is not None: - include_names = ", ".join(include_set) - print(f"Testing {entity_name}(s): {include_names}") - else: - include_names = None - - if reload: - doctest_latex_data_path = settings.get_doctest_latex_data_path( - should_be_readable=True - ) - output_data = load_doctest_data(doctest_latex_data_path) + print() + if index == 0: + print_and_log(f"No chapters found named {chapter_names}.") + elif failed > 0: + if not (keep_going and format == "latex"): + print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else "")) else: - output_data = {} - - # For consistency set the character encoding ASCII which is - # the lowest common denominator available on all systems. - settings.SYSTEM_CHARACTER_ENCODING = "ASCII" - - if DEFINITIONS is None: - print_and_log(LOGFILE, "Definitions are not initialized.") - return INVALID_TEST_GROUP_SETUP - - # Start with a clean variables state from whatever came before. - # In the test suite however, we may set new variables. - DEFINITIONS.reset_user_definitions() - return output_data, include_names - - -def test_tests( - index: int, - quiet: bool = False, - stop_on_failure: bool = False, - start_at: int = 0, - max_tests: int = MAX_TESTS, - excludes: Set[str] = set(), - generate_output: bool = False, - reload: bool = False, - keep_going: bool = False, -) -> Tuple[int, int, int, set, int]: - """ - Runs a group of related tests, ``Tests`` provided that the section is not listed in ``excludes`` and - the global test count given in ``index`` is not before ``start_at``. - - Tests are from a section or subsection (when the section is a guide section), - - If ``quiet`` is True, the progress and results of the tests are shown. - - ``index`` has the current count. We will stop on the first failure if ``stop_on_failure`` is true. - - """ - - total = index = failed = skipped = 0 - prev_key = [] - failed_symbols = set() - - output_data, names = validate_group_setup( - set(), - None, - reload, - ) - if (output_data, names) == INVALID_TEST_GROUP_SETUP: - return total, failed, skipped, failed_symbols, index - - for part in DOCUMENTATION.parts: - for chapter in part.chapters: - for section in chapter.all_sections: - section_name = section.title - if section_name in excludes: - continue - - if total >= max_tests: - break - ( - total, - failed, - prev_key, - ) = test_section_in_chapter( - section, - total, - failed, - quiet, - stop_on_failure, - prev_key, - start_at=start_at, - max_tests=max_tests, - ) - if generate_output and failed == 0: - create_output(section.doc.get_tests(), output_data) - pass - pass - - show_test_summary( - total, - failed, - "chapters", - names, - keep_going, - generate_output, - output_data, - ) - - if generate_output and (failed == 0 or keep_going): - save_doctest_data(output_data) - return total, failed, skipped, failed_symbols, index - - -def test_chapters( - include_chapters: set, - quiet=False, - stop_on_failure=False, - generate_output=False, - reload=False, - keep_going=False, - start_at: int = 0, - max_tests: int = MAX_TESTS, -) -> int: - """ - Runs a group of related tests for the set specified in ``chapters``. - - If ``quiet`` is True, the progress and results of the tests are shown. - - If ``stop_on_failure`` is true then the remaining tests in a section are skipped when a test - fails. - """ - failed = total = 0 - - output_data, chapter_names = validate_group_setup( - include_chapters, "chapters", reload - ) - if (output_data, chapter_names) == INVALID_TEST_GROUP_SETUP: - return total - - prev_key = [] - seen_chapters = set() - - for part in DOCUMENTATION.parts: - for chapter in part.chapters: - chapter_name = chapter.title - if chapter_name not in include_chapters: - continue - seen_chapters.add(chapter_name) - - for section in chapter.all_sections: - ( - total, - failed, - prev_key, - ) = test_section_in_chapter( - section, - total, - failed, - quiet, - stop_on_failure, - prev_key, - start_at=start_at, - max_tests=max_tests, - ) - if generate_output and failed == 0: - create_output(section.doc.get_tests(), output_data) - pass - pass - - if seen_chapters == include_chapters: - break - if chapter_name in include_chapters: - seen_chapters.add(chapter_name) - pass - - show_test_summary( - total, - failed, - "chapters", - chapter_names, - keep_going, - generate_output, - output_data, - ) - return total + print_and_log("All tests passed.") def test_sections( - include_sections: set, + sections: set, quiet=False, stop_on_failure=False, generate_output=False, reload=False, keep_going=False, -) -> int: - """Runs a group of related tests for the set specified in ``sections``. - - If ``quiet`` is True, the progress and results of the tests are shown. - - ``index`` has the current count. If ``stop_on_failure`` is true - then the remaining tests in a section are skipped when a test - fails. If ``keep_going`` is True and there is a failure, the next - section is continued after failure occurs. - """ - - total = failed = 0 - prev_key = [] - - output_data, section_names = validate_group_setup( - include_sections, "section", reload - ) - if (output_data, section_names) == INVALID_TEST_GROUP_SETUP: - return total - - seen_sections = set() - seen_last_section = False - last_section_name = None - section_name_for_finish = None +): + failed = 0 + index = 0 + section_names = ", ".join(sections) + print(f"Testing section(s): {section_names}") + sections |= {"$" + s for s in sections} + output_data = load_doctest_data() if reload else {} prev_key = [] + format = "latex" if generate_output else "text" + for tests in documentation.get_tests(): + if tests.section in sections: + for test in tests.tests: + key = list(test.key)[1:-1] + if prev_key != key: + prev_key = key + print(f'Testing section: {" / ".join(key)}') + index = 0 + if test.ignore: + continue + index += 1 + if not test_case(test, tests, index, quiet=quiet, format=format): + failed += 1 + if stop_on_failure: + break + if generate_output and (failed == 0 or keep_going): + create_output(tests, output_data, format=format) - for part in DOCUMENTATION.parts: - for chapter in part.chapters: - for section in chapter.all_sections: - ( - total, - failed, - prev_key, - ) = test_section_in_chapter( - section=section, - total=total, - quiet=quiet, - failed=failed, - stop_on_failure=stop_on_failure, - prev_key=prev_key, - include_sections=include_sections, - ) + print() + if index == 0: + print_and_log(f"No sections found named {section_names}.") + elif failed > 0: + if not (keep_going and format == "latex"): + print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else "")) + else: + print_and_log("All tests passed.") + if generate_output and (failed == 0 or keep_going): + save_doctest_data(output_data) - if generate_output and failed == 0: - create_output(section.doc.get_tests(), output_data) - pass - if last_section_name != section_name_for_finish: - if seen_sections == include_sections: - seen_last_section = True - break - if section_name_for_finish in include_sections: - seen_sections.add(section_name_for_finish) - last_section_name = section_name_for_finish - pass - - if seen_last_section: - break - pass - - show_test_summary( - total, - failed, - "sections", - section_names, - keep_going, - generate_output, - output_data, - ) - return total +def open_ensure_dir(f, *args, **kwargs): + try: + return open(f, *args, **kwargs) + except (IOError, OSError): + d = osp.dirname(f) + if d and not osp.exists(d): + os.makedirs(d) + return open(f, *args, **kwargs) def test_all( @@ -693,11 +348,11 @@ def test_all( generate_output=True, stop_on_failure=False, start_at=0, - max_tests: int = MAX_TESTS, + count=MAX_TESTS, texdatafolder=None, doc_even_if_error=False, - excludes: set = set(), -) -> int: + excludes=[], +): if not quiet: print(f"Testing {version_string}") @@ -708,66 +363,79 @@ def test_all( should_be_readable=False, create_parent=True ) ) - - total = failed = skipped = 0 try: index = 0 + total = failed = skipped = 0 failed_symbols = set() output_data = {} - sub_total, sub_failed, sub_skipped, symbols, index = test_tests( - index, - quiet=quiet, - stop_on_failure=stop_on_failure, - start_at=start_at, - max_tests=max_tests, - excludes=excludes, - generate_output=generate_output, - reload=False, - keep_going=not stop_on_failure, - ) - - total += sub_total - failed += sub_failed - skipped += sub_skipped - failed_symbols.update(symbols) - builtin_total = len(_builtins) + for tests in documentation.get_tests(): + sub_total, sub_failed, sub_skipped, symbols, index = test_tests( + tests, + index, + quiet=quiet, + stop_on_failure=stop_on_failure, + start_at=start_at, + max_tests=count, + excludes=excludes, + ) + if generate_output: + create_output(tests, output_data) + total += sub_total + failed += sub_failed + skipped += sub_skipped + failed_symbols.update(symbols) + if sub_failed and stop_on_failure: + break + if total >= count: + break + builtin_total = len(builtins) except KeyboardInterrupt: print("\nAborted.\n") - return total + return if failed > 0: - print(SEP) - if max_tests == MAX_TESTS: + print(sep) + if count == MAX_TESTS: print_and_log( - LOGFILE, - f"{total} Tests for {builtin_total} built-in symbols, {total-failed} " - f"passed, {failed} failed, {skipped} skipped.", + "%d Tests for %d built-in symbols, %d passed, %d failed, %d skipped." + % (total, builtin_total, total - failed - skipped, failed, skipped) ) else: print_and_log( - LOGFILE, - f"{total} Tests, {total - failed} passed, {failed} failed, {skipped} " - "skipped.", + "%d Tests, %d passed, %d failed, %d skipped." + % (total, total - failed, failed, skipped) ) if failed_symbols: if stop_on_failure: - print_and_log( - LOGFILE, "(not all tests are accounted for due to --stop-on-failure)" - ) - print_and_log(LOGFILE, "Failed:") + print_and_log("(not all tests are accounted for due to --stop-on-failure)") + print_and_log("Failed:") for part, chapter, section in sorted(failed_symbols): - print_and_log(LOGFILE, f" - {section} in {part} / {chapter}") + print_and_log(" - %s in %s / %s" % (section, part, chapter)) if generate_output and (failed == 0 or doc_even_if_error): save_doctest_data(output_data) - return total + return True if failed == 0: print("\nOK") else: print("\nFAILED") - sys.exit(1) # Travis-CI knows the tests have failed - return total + return sys.exit(1) # Travis-CI knows the tests have failed + + +def load_doctest_data() -> Dict[tuple, dict]: + """ + Load doctest tests and test results from Python PCL file. + + See ``save_doctest_data()`` for the format of the loaded PCL data + (a dict). + """ + doctest_latex_data_path = settings.get_doctest_latex_data_path( + should_be_readable=True + ) + print(f"Loading internal doctest data from {doctest_latex_data_path}") + with open_ensure_dir(doctest_latex_data_path, "rb") as doctest_data_file: + return pickle.load(doctest_data_file) def save_doctest_data(output_data: Dict[tuple, dict]): @@ -809,7 +477,7 @@ def write_doctest_data(quiet=False, reload=False): try: output_data = load_doctest_data() if reload else {} - for tests in DOCUMENTATION.get_tests(): + for tests in documentation.get_tests(): create_output(tests, output_data) except KeyboardInterrupt: print("\nAborted.\n") @@ -820,12 +488,12 @@ def write_doctest_data(quiet=False, reload=False): def main(): - global DEFINITIONS - global LOGFILE - global CHECK_PARTIAL_ELAPSED_TIME + global definitions + global logfile + global check_partial_elapsed_time import_and_load_builtins() - DEFINITIONS = Definitions(add_builtin=True) + definitions = Definitions(add_builtin=True) parser = ArgumentParser(description="Mathics test suite.", add_help=False) parser.add_argument( @@ -856,7 +524,7 @@ def main(): default="", dest="exclude", metavar="SECTION", - help="exclude SECTION(s). " + help="excude SECTION(s). " "You can list multiple sections by adding a comma (and no space) in between section names.", ) parser.add_argument( @@ -937,25 +605,25 @@ def main(): action="store_true", help="print cache statistics", ) - global LOGFILE + global logfile args = parser.parse_args() if args.elapsed_times: - CHECK_PARTIAL_ELAPSED_TIME = True + check_partial_elapsed_time = True # If a test for a specific section is called # just test it if args.logfilename: - LOGFILE = open(args.logfilename, "wt") + logfile = open(args.logfilename, "wt") - global DOCUMENTATION - DOCUMENTATION = MathicsMainDocumentation() + global documentation + documentation = MathicsMainDocumentation() # LoadModule Mathics3 modules if args.pymathics: for module_name in args.pymathics.split(","): try: - eval_LoadModule(module_name, DEFINITIONS) + eval_LoadModule(module_name, definitions) except PyMathicsLoadException: print(f"Python module {module_name} is not a Mathics3 module.") @@ -964,34 +632,23 @@ def main(): else: print(f"Mathics3 Module {module_name} loaded") - DOCUMENTATION.load_documentation_sources() - - start_time = None - total = 0 + documentation.load_documentation_sources() if args.sections: - include_sections = set(args.sections.split(",")) + sections = set(args.sections.split(",")) - start_time = datetime.now() - total = test_sections( - include_sections, + test_sections( + sections, stop_on_failure=args.stop_on_failure, generate_output=args.output, reload=args.reload, keep_going=args.keep_going, ) elif args.chapters: - start_time = datetime.now() - start_at = args.skip + 1 - include_chapters = set(args.chapters.split(",")) + chapters = set(args.chapters.split(",")) - total = test_chapters( - include_chapters, - stop_on_failure=args.stop_on_failure, - generate_output=args.output, - reload=args.reload, - start_at=start_at, - max_tests=args.count, + test_chapters( + chapters, stop_on_failure=args.stop_on_failure, reload=args.reload ) else: if args.doc_only: @@ -1003,24 +660,19 @@ def main(): excludes = set(args.exclude.split(",")) start_at = args.skip + 1 start_time = datetime.now() - total = test_all( + test_all( quiet=args.quiet, generate_output=args.output, stop_on_failure=args.stop_on_failure, start_at=start_at, - max_tests=args.count, + count=args.count, doc_even_if_error=args.keep_going, excludes=excludes, ) - pass - pass - - if total > 0 and start_time is not None: - end_time = datetime.now() - print("Test evalation took ", end_time - start_time) - - if LOGFILE: - LOGFILE.close() + end_time = datetime.now() + print("Tests took ", end_time - start_time) + if logfile: + logfile.close() if args.show_statistics: show_lru_cache_statistics() From 0ee92f45c9166a3f7d0076ec2ee1cc7fe34b1ed9 Mon Sep 17 00:00:00 2001 From: mmatera Date: Sun, 11 Feb 2024 10:17:20 -0300 Subject: [PATCH 5/6] upgrade docpipeline --- mathics/docpipeline.py | 536 ++++++++++++++++++++++++++--------------- 1 file changed, 341 insertions(+), 195 deletions(-) diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py index 7c1e36586..afb46a09f 100644 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# FIXME: combine with same thing in Mathics core +# FIXME: combine with same thing in Mathics Django """ Does 2 things which can either be done independently or as a pipeline: @@ -20,51 +20,45 @@ from typing import Dict, Optional import mathics -import mathics.settings from mathics import settings, version_string from mathics.core.definitions import Definitions from mathics.core.evaluation import Evaluation, Output -from mathics.core.load_builtin import ( - builtins_by_module, - builtins_dict, - import_and_load_builtins, -) +from mathics.core.load_builtin import _builtins, import_and_load_builtins from mathics.core.parser import MathicsSingleLineFeeder -from mathics.doc.common_doc import MathicsMainDocumentation +from mathics.doc.common_doc import ( + DocGuideSection, + DocSection, + DocTest, + DocTests, + MathicsMainDocumentation, +) +from mathics.doc.utils import load_doctest_data, print_and_log from mathics.eval.pymathics import PyMathicsLoadException, eval_LoadModule from mathics.timing import show_lru_cache_statistics -builtins = builtins_dict(builtins_by_module) - class TestOutput(Output): def max_stored_size(self, _): return None -sep = "-" * 70 + "\n" - # Global variables -definitions = None -documentation = None -check_partial_elapsed_time = False -logfile = None - -MAX_TESTS = 100000 # Number than the total number of tests +# FIXME: After 3.8 is the minimum Python we can turn "str" into a Literal +SEP: str = "-" * 70 + "\n" +STARS: str = "*" * 10 +DEFINITIONS = None +DOCUMENTATION = None +CHECK_PARTIAL_ELAPSED_TIME = False +LOGFILE = None -def print_and_log(*args): - a = [a.decode("utf-8") if isinstance(a, bytes) else a for a in args] - string = "".join(a) - print(string) - if logfile: - logfile.write(string) +MAX_TESTS = 100000 # A number greater than the total number of tests. -def compare(result: Optional[str], wanted: Optional[str]) -> bool: +def doctest_compare(result: Optional[str], wanted: Optional[str]) -> bool: """ - Performs test comparision betewen ``result`` and ``wanted`` and returns + Performs a doctest comparison between ``result`` and ``wanted`` and returns True if the test should be considered a success. """ if wanted in ("...", result): @@ -72,57 +66,70 @@ def compare(result: Optional[str], wanted: Optional[str]) -> bool: if result is None or wanted is None: return False - result = result.splitlines() - wanted = wanted.splitlines() - if result == [] and wanted == ["#<--#"]: + result_list = result.splitlines() + wanted_list = wanted.splitlines() + if result_list == [] and wanted_list == ["#<--#"]: return True - if len(result) != len(wanted): + if len(result_list) != len(wanted_list): return False - for r, w in zip(result, wanted): - wanted_re = re.escape(w.strip()) + for res, want in zip(result_list, wanted_list): + wanted_re = re.escape(want.strip()) wanted_re = wanted_re.replace("\\.\\.\\.", ".*?") wanted_re = f"^{wanted_re}$" - if not re.match(wanted_re, r.strip()): + if not re.match(wanted_re, res.strip()): return False return True -stars = "*" * 10 - - def test_case( - test, tests, index=0, subindex=0, quiet=False, section=None, format="text" + test: DocTest, + index: int = 0, + subindex: int = 0, + quiet: bool = False, + section_name: str = "", + section_for_print="", + chapter_name: str = "", + part: str = "", ) -> bool: - global check_partial_elapsed_time - test, wanted_out, wanted = test.test, test.outs, test.result + """ + Run a single test cases ``test``. Return True if test succeeds and False if it + fails. ``index``gives the global test number count, while ``subindex`` counts + from the beginning of the section or subsection. + + The test results are assumed to be foramtted to ASCII text. + """ + assert isinstance(index, int) + assert isinstance(subindex, int) + global CHECK_PARTIAL_ELAPSED_TIME + test_str, wanted_out, wanted = test.test, test.outs, test.result def fail(why): - part, chapter, section = tests.part, tests.chapter, tests.section print_and_log( - f"""{sep}Test failed: {section} in {part} / {chapter} + LOGFILE, + f"""{SEP}Test failed: in {part} / {chapter_name} / {section_name} {part} {why} """.encode( "utf-8" - ) + ), ) return False if not quiet: - if section: - print(f"{stars} {tests.chapter} / {section} {stars}".encode("utf-8")) - print(f"{index:4d} ({subindex:2d}): TEST {test}".encode("utf-8")) + if section_for_print: + print(f"{STARS} {section_for_print} {STARS}") + print(f"{index:4d} ({subindex:2d}): TEST {test_str}") - feeder = MathicsSingleLineFeeder(test, "") + feeder = MathicsSingleLineFeeder(test_str, filename="") evaluation = Evaluation( - definitions, catch_interrupt=False, output=TestOutput(), format=format + DEFINITIONS, catch_interrupt=False, output=TestOutput(), format="text" ) try: time_parsing = datetime.now() query = evaluation.parse_feeder(feeder) - if check_partial_elapsed_time: + if CHECK_PARTIAL_ELAPSED_TIME: print(" parsing took", datetime.now() - time_parsing) if query is None: # parsed expression is None @@ -130,24 +137,24 @@ def fail(why): out = evaluation.out else: result = evaluation.evaluate(query) - if check_partial_elapsed_time: + if CHECK_PARTIAL_ELAPSED_TIME: print(" evaluation took", datetime.now() - time_parsing) out = result.out result = result.result except Exception as exc: - fail("Exception %s" % exc) + fail(f"Exception {exc}") info = sys.exc_info() sys.excepthook(*info) return False time_comparing = datetime.now() - comparison_result = compare(result, wanted) + comparison_result = doctest_compare(result, wanted) - if check_partial_elapsed_time: + if CHECK_PARTIAL_ELAPSED_TIME: print(" comparison took ", datetime.now() - time_comparing) if not comparison_result: - print("result =!=wanted") - fail_msg = "Result: %s\nWanted: %s" % (result, wanted) + print("result != wanted") + fail_msg = f"Result: {result}\nWanted: {wanted}" if out: fail_msg += "\nAdditional output:\n" fail_msg += "\n".join(str(o) for o in out) @@ -158,7 +165,7 @@ def fail(why): # If we have ... don't check pass elif len(out) != len(wanted_out): - # Mismatched number of output lines and we don't have "..." + # Mismatched number of output lines, and we don't have "..." output_ok = False else: # Need to check all output line by line @@ -166,7 +173,7 @@ def fail(why): if not got == wanted and wanted.text != "...": output_ok = False break - if check_partial_elapsed_time: + if CHECK_PARTIAL_ELAPSED_TIME: print(" comparing messages took ", datetime.now() - time_comparing) if not output_ok: return fail( @@ -176,67 +183,33 @@ def fail(why): return True -def test_tests( - tests, - index, - quiet=False, - stop_on_failure=False, - start_at=0, - max_tests=MAX_TESTS, - excludes=[], -): - # For consistency set the character encoding ASCII which is - # the lowest common denominator available on all systems. - mathics.settings.SYSTEM_CHARACTER_ENCODING = "ASCII" - - definitions.reset_user_definitions() - total = failed = skipped = 0 - failed_symbols = set() - section = tests.section - if section in excludes: - return total, failed, len(tests.tests), failed_symbols, index - count = 0 - for subindex, test in enumerate(tests.tests): - index += 1 - if test.ignore: - continue - if index < start_at: - skipped += 1 - continue - elif count >= max_tests: - break - - total += 1 - count += 1 - if not test_case(test, tests, index, subindex + 1, quiet, section): - failed += 1 - failed_symbols.add((tests.part, tests.chapter, tests.section)) - if stop_on_failure: - break - - section = None - return total, failed, skipped, failed_symbols, index +def create_output(tests, doctest_data, output_format="latex"): + """ + Populate ``doctest_data`` with the results of the + ``tests`` in the format ``output_format`` + """ + if DEFINITIONS is None: + print_and_log(LOGFILE, "Definitions are not initialized.") + return + DEFINITIONS.reset_user_definitions() -# FIXME: move this to common routine -def create_output(tests, doctest_data, format="latex"): - definitions.reset_user_definitions() for test in tests.tests: if test.private: continue key = test.key evaluation = Evaluation( - definitions, format=format, catch_interrupt=True, output=TestOutput() + DEFINITIONS, format=output_format, catch_interrupt=True, output=TestOutput() ) try: result = evaluation.parse_evaluate(test.test) - except: # noqa + except Exception: # noqa result = None if result is None: result = [] else: result_data = result.get_data() - result_data["form"] = format + result_data["form"] = output_format result = [result_data] doctest_data[key] = { @@ -245,22 +218,73 @@ def create_output(tests, doctest_data, format="latex"): } +def show_test_summary( + total: int, + failed: int, + entity_name: str, + entities_searched: str, + keep_going: bool, + generate_output: bool, + output_data, +): + """ + Print and log test summary results. + + If ``generate_output`` is True, we will also generate output data + to ``output_data``. + """ + + print() + if total == 0: + print_and_log( + LOGFILE, f"No {entity_name} found with a name in: {entities_searched}." + ) + if "MATHICS_DEBUG_TEST_CREATE" not in os.environ: + print(f"Set environment MATHICS_DEBUG_TEST_CREATE to see {entity_name}.") + elif failed > 0: + print(SEP) + if not generate_output: + print_and_log( + LOGFILE, f"""{failed} test{'s' if failed != 1 else ''} failed.""" + ) + else: + print_and_log(LOGFILE, "All tests passed.") + + if generate_output and (failed == 0 or keep_going): + save_doctest_data(output_data) + return + + def test_chapters( - chapters: set, + include_chapters: set, quiet=False, stop_on_failure=False, generate_output=False, reload=False, keep_going=False, -): - failed = 0 + start_at: int = 0, + max_tests: int = MAX_TESTS, +) -> int: + """ + Runs a group of related tests for the set specified in ``chapters``. + + If ``quiet`` is True, the progress and results of the tests are shown. + + If ``stop_on_failure`` is true then the remaining tests in a section are skipped when a test + fails. + """ + failed = total = 0 index = 0 - chapter_names = ", ".join(chapters) - print(f"Testing chapter(s): {chapter_names}") - output_data = load_doctest_data() if reload else {} + + output_data, chapter_names = validate_group_setup( + include_chapters, "chapters", reload + ) + if (output_data, chapter_names) == INVALID_TEST_GROUP_SETUP: + return total + prev_key = [] - for tests in documentation.get_tests(): - if tests.chapter in chapters: + for tests in DOCUMENTATION.get_tests(): + if tests.chapter in include_chapters: for test in tests.tests: key = list(test.key)[1:-1] if prev_key != key: @@ -270,7 +294,8 @@ def test_chapters( if test.ignore: continue index += 1 - if not test_case(test, tests, index, quiet=quiet): + total += 1 + if not test_case(test, index, quiet=quiet): failed += 1 if stop_on_failure: break @@ -279,32 +304,56 @@ def test_chapters( print() if index == 0: - print_and_log(f"No chapters found named {chapter_names}.") + print_and_log(LOGFILE, f"No chapters found named {chapter_names}.") elif failed > 0: if not (keep_going and format == "latex"): - print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else "")) + print_and_log( + LOGFILE, "%d test%s failed." % (failed, "s" if failed != 1 else "") + ) else: - print_and_log("All tests passed.") + print_and_log(LOGFILE, "All tests passed.") + + show_test_summary( + total, + failed, + "chapters", + chapter_names, + keep_going, + generate_output, + output_data, + ) + return total def test_sections( - sections: set, + include_sections: set, quiet=False, stop_on_failure=False, generate_output=False, reload=False, keep_going=False, -): - failed = 0 - index = 0 - section_names = ", ".join(sections) - print(f"Testing section(s): {section_names}") - sections |= {"$" + s for s in sections} - output_data = load_doctest_data() if reload else {} +) -> int: + """Runs a group of related tests for the set specified in ``sections``. + + If ``quiet`` is True, the progress and results of the tests are shown. + + ``index`` has the current count. If ``stop_on_failure`` is true + then the remaining tests in a section are skipped when a test + fails. If ``keep_going`` is True and there is a failure, the next + section is continued after failure occurs. + """ + + total = failed = 0 prev_key = [] + + output_data, section_names = validate_group_setup( + include_sections, "section", reload + ) + + index = 0 format = "latex" if generate_output else "text" - for tests in documentation.get_tests(): - if tests.section in sections: + for tests in DOCUMENTATION.get_tests(): + if tests.section in include_sections: for test in tests.tests: key = list(test.key)[1:-1] if prev_key != key: @@ -314,7 +363,8 @@ def test_sections( if test.ignore: continue index += 1 - if not test_case(test, tests, index, quiet=quiet, format=format): + total += 1 + if not test_case(test, index, quiet=quiet): failed += 1 if stop_on_failure: break @@ -323,24 +373,27 @@ def test_sections( print() if index == 0: - print_and_log(f"No sections found named {section_names}.") + print_and_log(LOGFILE, f"No sections found named {section_names}.") elif failed > 0: if not (keep_going and format == "latex"): - print_and_log("%d test%s failed." % (failed, "s" if failed != 1 else "")) + print_and_log( + LOGFILE, "%d test%s failed." % (failed, "s" if failed != 1 else "") + ) else: - print_and_log("All tests passed.") + print_and_log(LOGFILE, "All tests passed.") if generate_output and (failed == 0 or keep_going): save_doctest_data(output_data) - -def open_ensure_dir(f, *args, **kwargs): - try: - return open(f, *args, **kwargs) - except (IOError, OSError): - d = osp.dirname(f) - if d and not osp.exists(d): - os.makedirs(d) - return open(f, *args, **kwargs) + show_test_summary( + total, + failed, + "sections", + section_names, + keep_going, + generate_output, + output_data, + ) + return total def test_all( @@ -348,11 +401,11 @@ def test_all( generate_output=True, stop_on_failure=False, start_at=0, - count=MAX_TESTS, + max_tests: int = MAX_TESTS, texdatafolder=None, doc_even_if_error=False, excludes=[], -): +) -> int: if not quiet: print(f"Testing {version_string}") @@ -368,14 +421,14 @@ def test_all( total = failed = skipped = 0 failed_symbols = set() output_data = {} - for tests in documentation.get_tests(): + for tests in DOCUMENTATION.get_tests(): sub_total, sub_failed, sub_skipped, symbols, index = test_tests( tests, index, quiet=quiet, stop_on_failure=stop_on_failure, start_at=start_at, - max_tests=count, + max_tests=max_tests, excludes=excludes, ) if generate_output: @@ -386,56 +439,46 @@ def test_all( failed_symbols.update(symbols) if sub_failed and stop_on_failure: break - if total >= count: + if total >= max_tests: break - builtin_total = len(builtins) + builtin_total = len(_builtins) except KeyboardInterrupt: print("\nAborted.\n") - return + return total if failed > 0: - print(sep) - if count == MAX_TESTS: + print(SEP) + if max_tests == MAX_TESTS: print_and_log( - "%d Tests for %d built-in symbols, %d passed, %d failed, %d skipped." - % (total, builtin_total, total - failed - skipped, failed, skipped) + LOGFILE, + f"{total} Tests for {builtin_total} built-in symbols, {total-failed} " + f"passed, {failed} failed, {skipped} skipped.", ) else: print_and_log( - "%d Tests, %d passed, %d failed, %d skipped." - % (total, total - failed, failed, skipped) + LOGFILE, + f"{total} Tests, {total - failed} passed, {failed} failed, {skipped} " + "skipped.", ) if failed_symbols: if stop_on_failure: - print_and_log("(not all tests are accounted for due to --stop-on-failure)") - print_and_log("Failed:") + print_and_log( + LOGFILE, "(not all tests are accounted for due to --stop-on-failure)" + ) + print_and_log(LOGFILE, "Failed:") for part, chapter, section in sorted(failed_symbols): - print_and_log(" - %s in %s / %s" % (section, part, chapter)) + print_and_log(LOGFILE, f" - {section} in {part} / {chapter}") if generate_output and (failed == 0 or doc_even_if_error): save_doctest_data(output_data) - return True + return total if failed == 0: print("\nOK") else: print("\nFAILED") - return sys.exit(1) # Travis-CI knows the tests have failed - - -def load_doctest_data() -> Dict[tuple, dict]: - """ - Load doctest tests and test results from Python PCL file. - - See ``save_doctest_data()`` for the format of the loaded PCL data - (a dict). - """ - doctest_latex_data_path = settings.get_doctest_latex_data_path( - should_be_readable=True - ) - print(f"Loading internal doctest data from {doctest_latex_data_path}") - with open_ensure_dir(doctest_latex_data_path, "rb") as doctest_data_file: - return pickle.load(doctest_data_file) + sys.exit(1) # Travis-CI knows the tests have failed + return total def save_doctest_data(output_data: Dict[tuple, dict]): @@ -466,6 +509,93 @@ def save_doctest_data(output_data: Dict[tuple, dict]): pickle.dump(output_data, output_file, 4) +# When 3.8 is base, the below can be a Literal type. +INVALID_TEST_GROUP_SETUP = (None, None) + + +def validate_group_setup( + include_set: set, + entity_name: Optional[str], + reload: bool, +) -> tuple: + """ + Common things that need to be done before running a group of doctests. + """ + + if DOCUMENTATION is None: + print_and_log(LOGFILE, "Documentation is not initialized.") + return INVALID_TEST_GROUP_SETUP + + if entity_name is not None: + include_names = ", ".join(include_set) + print(f"Testing {entity_name}(s): {include_names}") + else: + include_names = None + + if reload: + doctest_latex_data_path = settings.get_doctest_latex_data_path( + should_be_readable=True + ) + output_data = load_doctest_data(doctest_latex_data_path) + else: + output_data = {} + + # For consistency set the character encoding ASCII which is + # the lowest common denominator available on all systems. + settings.SYSTEM_CHARACTER_ENCODING = "ASCII" + + if DEFINITIONS is None: + print_and_log(LOGFILE, "Definitions are not initialized.") + return INVALID_TEST_GROUP_SETUP + + # Start with a clean variables state from whatever came before. + # In the test suite however, we may set new variables. + DEFINITIONS.reset_user_definitions() + return output_data, include_names + + +def test_tests( + tests, + index, + quiet=False, + stop_on_failure=False, + start_at=0, + max_tests=MAX_TESTS, + excludes=[], +): + # For consistency set the character encoding ASCII which is + # the lowest common denominator available on all systems. + settings.SYSTEM_CHARACTER_ENCODING = "ASCII" + DEFINITIONS.reset_user_definitions() + total = failed = skipped = 0 + failed_symbols = set() + section = tests.section + if section in excludes: + return total, failed, len(tests.tests), failed_symbols, index + count = 0 + print(tests.chapter, "/", section) + for subindex, test in enumerate(tests.tests): + index += 1 + if test.ignore: + continue + if index < start_at: + skipped += 1 + continue + elif count >= max_tests: + break + + total += 1 + count += 1 + if not test_case(test, index, subindex + 1, quiet, section): + failed += 1 + failed_symbols.add((tests.part, tests.chapter, tests.section)) + if stop_on_failure: + break + + section = None + return total, failed, skipped, failed_symbols, index + + def write_doctest_data(quiet=False, reload=False): """ Get doctest information, which involves running the tests to obtain @@ -477,7 +607,7 @@ def write_doctest_data(quiet=False, reload=False): try: output_data = load_doctest_data() if reload else {} - for tests in documentation.get_tests(): + for tests in DOCUMENTATION.get_tests(): create_output(tests, output_data) except KeyboardInterrupt: print("\nAborted.\n") @@ -488,12 +618,12 @@ def write_doctest_data(quiet=False, reload=False): def main(): - global definitions - global logfile - global check_partial_elapsed_time + global DEFINITIONS + global LOGFILE + global CHECK_PARTIAL_ELAPSED_TIME import_and_load_builtins() - definitions = Definitions(add_builtin=True) + DEFINITIONS = Definitions(add_builtin=True) parser = ArgumentParser(description="Mathics test suite.", add_help=False) parser.add_argument( @@ -524,7 +654,7 @@ def main(): default="", dest="exclude", metavar="SECTION", - help="excude SECTION(s). " + help="exclude SECTION(s). " "You can list multiple sections by adding a comma (and no space) in between section names.", ) parser.add_argument( @@ -605,25 +735,25 @@ def main(): action="store_true", help="print cache statistics", ) - global logfile + global LOGFILE args = parser.parse_args() if args.elapsed_times: - check_partial_elapsed_time = True + CHECK_PARTIAL_ELAPSED_TIME = True # If a test for a specific section is called # just test it if args.logfilename: - logfile = open(args.logfilename, "wt") + LOGFILE = open(args.logfilename, "wt") - global documentation - documentation = MathicsMainDocumentation() + global DOCUMENTATION + DOCUMENTATION = MathicsMainDocumentation() # LoadModule Mathics3 modules if args.pymathics: for module_name in args.pymathics.split(","): try: - eval_LoadModule(module_name, definitions) + eval_LoadModule(module_name, DEFINITIONS) except PyMathicsLoadException: print(f"Python module {module_name} is not a Mathics3 module.") @@ -632,24 +762,36 @@ def main(): else: print(f"Mathics3 Module {module_name} loaded") - documentation.load_documentation_sources() + DOCUMENTATION.load_documentation_sources() + + start_time = None + total = 0 if args.sections: - sections = set(args.sections.split(",")) + include_sections = set(sec.strip() for sec in args.sections.split(",")) - test_sections( - sections, + total = test_sections( + include_sections, stop_on_failure=args.stop_on_failure, generate_output=args.output, reload=args.reload, keep_going=args.keep_going, ) + assert isinstance(total, int) elif args.chapters: - chapters = set(args.chapters.split(",")) + start_time = datetime.now() + start_at = args.skip + 1 + include_chapters = set(chap.strip() for chap in args.chapters.split(",")) - test_chapters( - chapters, stop_on_failure=args.stop_on_failure, reload=args.reload + total = test_chapters( + include_chapters, + stop_on_failure=args.stop_on_failure, + generate_output=args.output, + reload=args.reload, + start_at=start_at, + max_tests=args.count, ) + assert isinstance(total, int) else: if args.doc_only: write_doctest_data( @@ -660,19 +802,23 @@ def main(): excludes = set(args.exclude.split(",")) start_at = args.skip + 1 start_time = datetime.now() - test_all( + total = test_all( quiet=args.quiet, generate_output=args.output, stop_on_failure=args.stop_on_failure, start_at=start_at, - count=args.count, + max_tests=args.count, doc_even_if_error=args.keep_going, excludes=excludes, ) - end_time = datetime.now() - print("Tests took ", end_time - start_time) - if logfile: - logfile.close() + assert isinstance(total, int) + + end_time = datetime.now() + if total > 0 and start_time is not None: + print("Tests took ", end_time - start_time) + + if LOGFILE: + LOGFILE.close() if args.show_statistics: show_lru_cache_statistics() From ab369b15adeede542356e9ebc3564a630dad1483 Mon Sep 17 00:00:00 2001 From: mmatera Date: Wed, 13 Mar 2024 16:42:51 -0300 Subject: [PATCH 6/6] restore docpipeline --- mathics/docpipeline.py | 565 ++++++++++++++++++++++++++++------------- 1 file changed, 383 insertions(+), 182 deletions(-) mode change 100644 => 100755 mathics/docpipeline.py diff --git a/mathics/docpipeline.py b/mathics/docpipeline.py old mode 100644 new mode 100755 index afb46a09f..f75fb2310 --- a/mathics/docpipeline.py +++ b/mathics/docpipeline.py @@ -17,7 +17,7 @@ import sys from argparse import ArgumentParser from datetime import datetime -from typing import Dict, Optional +from typing import Dict, Optional, Set, Tuple, Union import mathics from mathics import settings, version_string @@ -43,7 +43,6 @@ def max_stored_size(self, _): # Global variables - # FIXME: After 3.8 is the minimum Python we can turn "str" into a Literal SEP: str = "-" * 70 + "\n" STARS: str = "*" * 10 @@ -53,6 +52,7 @@ def max_stored_size(self, _): CHECK_PARTIAL_ELAPSED_TIME = False LOGFILE = None + MAX_TESTS = 100000 # A number greater than the total number of tests. @@ -66,6 +66,7 @@ def doctest_compare(result: Optional[str], wanted: Optional[str]) -> bool: if result is None or wanted is None: return False + result_list = result.splitlines() wanted_list = wanted.splitlines() if result_list == [] and wanted_list == ["#<--#"]: @@ -100,8 +101,7 @@ def test_case( The test results are assumed to be foramtted to ASCII text. """ - assert isinstance(index, int) - assert isinstance(subindex, int) + global CHECK_PARTIAL_ELAPSED_TIME test_str, wanted_out, wanted = test.test, test.outs, test.result @@ -152,6 +152,7 @@ def fail(why): if CHECK_PARTIAL_ELAPSED_TIME: print(" comparison took ", datetime.now() - time_comparing) + if not comparison_result: print("result != wanted") fail_msg = f"Result: {result}\nWanted: {wanted}" @@ -188,13 +189,13 @@ def create_output(tests, doctest_data, output_format="latex"): Populate ``doctest_data`` with the results of the ``tests`` in the format ``output_format`` """ + if DEFINITIONS is None: print_and_log(LOGFILE, "Definitions are not initialized.") return DEFINITIONS.reset_user_definitions() - - for test in tests.tests: + for test in tests: if test.private: continue key = test.key @@ -255,6 +256,284 @@ def show_test_summary( return +def test_section_in_chapter( + section: Union[DocSection, DocGuideSection], + total: int, + failed: int, + quiet, + stop_on_failure, + prev_key: list, + include_sections: Optional[Set[str]] = None, + start_at: int = 0, + skipped: int = 0, + max_tests: int = MAX_TESTS, +) -> Tuple[int, int, list]: + """ + Runs a tests for section ``section`` under a chapter or guide section. + Note that both of these contain a collection of section tests underneath. + + ``total`` and ``failed`` give running tallies on the number of tests run and + the number of tests respectively. + + If ``quiet`` is True, the progress and results of the tests are shown. + If ``stop_on_failure`` is true then the remaining tests in a section are skipped when a test + fails. + """ + section_name = section.title + + # Start out assuming all subsections will be tested + include_subsections = None + + if include_sections is not None and section_name not in include_sections: + # use include_section to filter subsections + include_subsections = include_sections + + chapter = section.chapter + chapter_name = chapter.title + part_name = chapter.part.title + index = 0 + if len(section.subsections) > 0: + for subsection in section.subsections: + if ( + include_subsections is not None + and subsection.title not in include_subsections + ): + continue + + DEFINITIONS.reset_user_definitions() + for test in subsection.doc.get_tests(): + # Get key dropping off test index number + key = list(test.key)[1:-1] + if prev_key != key: + prev_key = key + section_name_for_print = " / ".join(key) + if quiet: + # We don't print with stars inside in test_case(), so print here. + print(f"Testing section: {section_name_for_print}") + index = 0 + else: + # Null out section name, so that on the next iteration we do not print a section header + # in test_case(). + section_name_for_print = "" + + if isinstance(test, DocTests): + for doctest in test.tests: + index += 1 + total += 1 + if not test_case( + doctest, + total, + index, + quiet=quiet, + section_name=section_name, + section_for_print=section_name_for_print, + chapter_name=chapter_name, + part=part_name, + ): + failed += 1 + if stop_on_failure: + break + elif test.ignore: + continue + + else: + index += 1 + + if index < start_at: + skipped += 1 + continue + + total += 1 + if not test_case( + test, + total, + index, + quiet=quiet, + section_name=section_name, + section_for_print=section_name_for_print, + chapter_name=chapter_name, + part=part_name, + ): + failed += 1 + if stop_on_failure: + break + pass + pass + pass + pass + else: + if include_subsections is None or section.title in include_subsections: + DEFINITIONS.reset_user_definitions() + for test in section.doc.get_tests(): + # Get key dropping off test index number + key = list(test.key)[1:-1] + if prev_key != key: + prev_key = key + section_name_for_print = " / ".join(key) + if quiet: + print(f"Testing section: {section_name_for_print}") + index = 0 + else: + # Null out section name, so that on the next iteration we do not print a section header. + section_name_for_print = "" + + if test.ignore: + continue + + else: + index += 1 + + if index < start_at: + skipped += 1 + continue + + total += 1 + if total >= max_tests: + break + + if not test_case( + test, + total, + index, + quiet=quiet, + section_name=section.title, + section_for_print=section_name_for_print, + chapter_name=chapter.title, + part=part_name, + ): + failed += 1 + if stop_on_failure: + break + pass + pass + + pass + return total, failed, prev_key + + +# When 3.8 is base, the below can be a Literal type. +INVALID_TEST_GROUP_SETUP = (None, None) + + +def validate_group_setup( + include_set: set, + entity_name: Optional[str], + reload: bool, +) -> tuple: + """ + Common things that need to be done before running a group of doctests. + """ + + if DOCUMENTATION is None: + print_and_log(LOGFILE, "Documentation is not initialized.") + return INVALID_TEST_GROUP_SETUP + + if entity_name is not None: + include_names = ", ".join(include_set) + print(f"Testing {entity_name}(s): {include_names}") + else: + include_names = None + + if reload: + doctest_latex_data_path = settings.get_doctest_latex_data_path( + should_be_readable=True + ) + output_data = load_doctest_data(doctest_latex_data_path) + else: + output_data = {} + + # For consistency set the character encoding ASCII which is + # the lowest common denominator available on all systems. + settings.SYSTEM_CHARACTER_ENCODING = "ASCII" + + if DEFINITIONS is None: + print_and_log(LOGFILE, "Definitions are not initialized.") + return INVALID_TEST_GROUP_SETUP + + # Start with a clean variables state from whatever came before. + # In the test suite however, we may set new variables. + DEFINITIONS.reset_user_definitions() + return output_data, include_names + + +def test_tests( + index: int, + quiet: bool = False, + stop_on_failure: bool = False, + start_at: int = 0, + max_tests: int = MAX_TESTS, + excludes: Set[str] = set(), + generate_output: bool = False, + reload: bool = False, + keep_going: bool = False, +) -> Tuple[int, int, int, set, int]: + """ + Runs a group of related tests, ``Tests`` provided that the section is not listed in ``excludes`` and + the global test count given in ``index`` is not before ``start_at``. + + Tests are from a section or subsection (when the section is a guide section), + + If ``quiet`` is True, the progress and results of the tests are shown. + + ``index`` has the current count. We will stop on the first failure if ``stop_on_failure`` is true. + + """ + + total = index = failed = skipped = 0 + prev_key = [] + failed_symbols = set() + + output_data, names = validate_group_setup( + set(), + None, + reload, + ) + if (output_data, names) == INVALID_TEST_GROUP_SETUP: + return total, failed, skipped, failed_symbols, index + + for part in DOCUMENTATION.parts: + for chapter in part.chapters: + for section in chapter.all_sections: + section_name = section.title + if section_name in excludes: + continue + + if total >= max_tests: + break + ( + total, + failed, + prev_key, + ) = test_section_in_chapter( + section, + total, + failed, + quiet, + stop_on_failure, + prev_key, + start_at=start_at, + max_tests=max_tests, + ) + if generate_output and failed == 0: + create_output(section.doc.get_tests(), output_data) + pass + pass + + show_test_summary( + total, + failed, + "chapters", + names, + keep_going, + generate_output, + output_data, + ) + + if generate_output and (failed == 0 or keep_going): + save_doctest_data(output_data) + return total, failed, skipped, failed_symbols, index + + def test_chapters( include_chapters: set, quiet=False, @@ -274,7 +553,6 @@ def test_chapters( fails. """ failed = total = 0 - index = 0 output_data, chapter_names = validate_group_setup( include_chapters, "chapters", reload @@ -283,35 +561,40 @@ def test_chapters( return total prev_key = [] - for tests in DOCUMENTATION.get_tests(): - if tests.chapter in include_chapters: - for test in tests.tests: - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - print(f'Testing section: {" / ".join(key)}') - index = 0 - if test.ignore: - continue - index += 1 - total += 1 - if not test_case(test, index, quiet=quiet): - failed += 1 - if stop_on_failure: - break - if generate_output and failed == 0: - create_output(tests, output_data) + seen_chapters = set() + + for part in DOCUMENTATION.parts: + for chapter in part.chapters: + chapter_name = chapter.title + if chapter_name not in include_chapters: + continue + seen_chapters.add(chapter_name) + + for section in chapter.all_sections: + ( + total, + failed, + prev_key, + ) = test_section_in_chapter( + section, + total, + failed, + quiet, + stop_on_failure, + prev_key, + start_at=start_at, + max_tests=max_tests, + ) + if generate_output and failed == 0: + create_output(section.doc.get_tests(), output_data) + pass + pass - print() - if index == 0: - print_and_log(LOGFILE, f"No chapters found named {chapter_names}.") - elif failed > 0: - if not (keep_going and format == "latex"): - print_and_log( - LOGFILE, "%d test%s failed." % (failed, "s" if failed != 1 else "") - ) - else: - print_and_log(LOGFILE, "All tests passed.") + if seen_chapters == include_chapters: + break + if chapter_name in include_chapters: + seen_chapters.add(chapter_name) + pass show_test_summary( total, @@ -349,40 +632,48 @@ def test_sections( output_data, section_names = validate_group_setup( include_sections, "section", reload ) + if (output_data, section_names) == INVALID_TEST_GROUP_SETUP: + return total - index = 0 - format = "latex" if generate_output else "text" - for tests in DOCUMENTATION.get_tests(): - if tests.section in include_sections: - for test in tests.tests: - key = list(test.key)[1:-1] - if prev_key != key: - prev_key = key - print(f'Testing section: {" / ".join(key)}') - index = 0 - if test.ignore: - continue - index += 1 - total += 1 - if not test_case(test, index, quiet=quiet): - failed += 1 - if stop_on_failure: + seen_sections = set() + seen_last_section = False + last_section_name = None + section_name_for_finish = None + prev_key = [] + + for part in DOCUMENTATION.parts: + for chapter in part.chapters: + for section in chapter.all_sections: + ( + total, + failed, + prev_key, + ) = test_section_in_chapter( + section=section, + total=total, + quiet=quiet, + failed=failed, + stop_on_failure=stop_on_failure, + prev_key=prev_key, + include_sections=include_sections, + ) + + if generate_output and failed == 0: + create_output(section.doc.get_tests(), output_data) + pass + + if last_section_name != section_name_for_finish: + if seen_sections == include_sections: + seen_last_section = True break - if generate_output and (failed == 0 or keep_going): - create_output(tests, output_data, format=format) + if section_name_for_finish in include_sections: + seen_sections.add(section_name_for_finish) + last_section_name = section_name_for_finish + pass - print() - if index == 0: - print_and_log(LOGFILE, f"No sections found named {section_names}.") - elif failed > 0: - if not (keep_going and format == "latex"): - print_and_log( - LOGFILE, "%d test%s failed." % (failed, "s" if failed != 1 else "") - ) - else: - print_and_log(LOGFILE, "All tests passed.") - if generate_output and (failed == 0 or keep_going): - save_doctest_data(output_data) + if seen_last_section: + break + pass show_test_summary( total, @@ -404,7 +695,7 @@ def test_all( max_tests: int = MAX_TESTS, texdatafolder=None, doc_even_if_error=False, - excludes=[], + excludes: set = set(), ) -> int: if not quiet: print(f"Testing {version_string}") @@ -416,31 +707,28 @@ def test_all( should_be_readable=False, create_parent=True ) ) + + total = failed = skipped = 0 try: index = 0 - total = failed = skipped = 0 failed_symbols = set() output_data = {} - for tests in DOCUMENTATION.get_tests(): - sub_total, sub_failed, sub_skipped, symbols, index = test_tests( - tests, - index, - quiet=quiet, - stop_on_failure=stop_on_failure, - start_at=start_at, - max_tests=max_tests, - excludes=excludes, - ) - if generate_output: - create_output(tests, output_data) - total += sub_total - failed += sub_failed - skipped += sub_skipped - failed_symbols.update(symbols) - if sub_failed and stop_on_failure: - break - if total >= max_tests: - break + sub_total, sub_failed, sub_skipped, symbols, index = test_tests( + index, + quiet=quiet, + stop_on_failure=stop_on_failure, + start_at=start_at, + max_tests=max_tests, + excludes=excludes, + generate_output=generate_output, + reload=False, + keep_going=not stop_on_failure, + ) + + total += sub_total + failed += sub_failed + skipped += sub_skipped + failed_symbols.update(symbols) builtin_total = len(_builtins) except KeyboardInterrupt: print("\nAborted.\n") @@ -509,93 +797,6 @@ def save_doctest_data(output_data: Dict[tuple, dict]): pickle.dump(output_data, output_file, 4) -# When 3.8 is base, the below can be a Literal type. -INVALID_TEST_GROUP_SETUP = (None, None) - - -def validate_group_setup( - include_set: set, - entity_name: Optional[str], - reload: bool, -) -> tuple: - """ - Common things that need to be done before running a group of doctests. - """ - - if DOCUMENTATION is None: - print_and_log(LOGFILE, "Documentation is not initialized.") - return INVALID_TEST_GROUP_SETUP - - if entity_name is not None: - include_names = ", ".join(include_set) - print(f"Testing {entity_name}(s): {include_names}") - else: - include_names = None - - if reload: - doctest_latex_data_path = settings.get_doctest_latex_data_path( - should_be_readable=True - ) - output_data = load_doctest_data(doctest_latex_data_path) - else: - output_data = {} - - # For consistency set the character encoding ASCII which is - # the lowest common denominator available on all systems. - settings.SYSTEM_CHARACTER_ENCODING = "ASCII" - - if DEFINITIONS is None: - print_and_log(LOGFILE, "Definitions are not initialized.") - return INVALID_TEST_GROUP_SETUP - - # Start with a clean variables state from whatever came before. - # In the test suite however, we may set new variables. - DEFINITIONS.reset_user_definitions() - return output_data, include_names - - -def test_tests( - tests, - index, - quiet=False, - stop_on_failure=False, - start_at=0, - max_tests=MAX_TESTS, - excludes=[], -): - # For consistency set the character encoding ASCII which is - # the lowest common denominator available on all systems. - settings.SYSTEM_CHARACTER_ENCODING = "ASCII" - DEFINITIONS.reset_user_definitions() - total = failed = skipped = 0 - failed_symbols = set() - section = tests.section - if section in excludes: - return total, failed, len(tests.tests), failed_symbols, index - count = 0 - print(tests.chapter, "/", section) - for subindex, test in enumerate(tests.tests): - index += 1 - if test.ignore: - continue - if index < start_at: - skipped += 1 - continue - elif count >= max_tests: - break - - total += 1 - count += 1 - if not test_case(test, index, subindex + 1, quiet, section): - failed += 1 - failed_symbols.add((tests.part, tests.chapter, tests.section)) - if stop_on_failure: - break - - section = None - return total, failed, skipped, failed_symbols, index - - def write_doctest_data(quiet=False, reload=False): """ Get doctest information, which involves running the tests to obtain @@ -768,8 +969,9 @@ def main(): total = 0 if args.sections: - include_sections = set(sec.strip() for sec in args.sections.split(",")) + include_sections = set(args.sections.split(",")) + start_time = datetime.now() total = test_sections( include_sections, stop_on_failure=args.stop_on_failure, @@ -777,11 +979,10 @@ def main(): reload=args.reload, keep_going=args.keep_going, ) - assert isinstance(total, int) elif args.chapters: start_time = datetime.now() start_at = args.skip + 1 - include_chapters = set(chap.strip() for chap in args.chapters.split(",")) + include_chapters = set(args.chapters.split(",")) total = test_chapters( include_chapters, @@ -791,7 +992,6 @@ def main(): start_at=start_at, max_tests=args.count, ) - assert isinstance(total, int) else: if args.doc_only: write_doctest_data( @@ -811,11 +1011,12 @@ def main(): doc_even_if_error=args.keep_going, excludes=excludes, ) - assert isinstance(total, int) + pass + pass - end_time = datetime.now() if total > 0 and start_time is not None: - print("Tests took ", end_time - start_time) + end_time = datetime.now() + print("Test evalation took ", end_time - start_time) if LOGFILE: LOGFILE.close()