Some refactoring

ad-freiburg · Oct 2, 2023 · 6d98955 · 6d98955
1 parent 6692efd
commit 6d98955
Show file tree

Hide file tree

Showing 10 changed files with 215 additions and 196 deletions.
diff --git a/Makefile b/Makefile
@@ -72,11 +72,12 @@ CXX							= $(CXX_PROD)
 
 # ==================================================================================================
 
-.PHONY: help checkstyle compile-debug compile-prod compile test unit-test e2e-test e2e-analyze \
-  install install-without-requirements clean release check-version set-version version packages \
-	apt-repo apt-repo/build-docker-image apt-repo/update apt-repo/server/start \
-	apt-repo/server/start/force apt-repo/server/stop requirements/checkstyle requirements/compile \
-	requirements/test requirements/install requirements/packages
+.PHONY: help checkstyle compile-debug compile-prod compile test unit-test e2e-test \
+  e2e-analyze-latest e2e-analyze-vscode install install-without-requirements clean release \
+	check-version set-version version packages apt-repo apt-repo/build-docker-image \
+	apt-repo/update apt-repo/server/start apt-repo/server/start/force apt-repo/server/stop \
+	requirements/checkstyle requirements/compile requirements/test requirements/install \
+	requirements/packages
 
 # --------------------------------------------------------------------------------------------------
 # Help.
@@ -145,13 +146,16 @@ $(BUILD_DIR)/%Test.o: %Test.cpp $(SRC_HEADER_FILES)
 	$(CXX_TEST) -c $< -o "$@" $(CXX_LIBS_TEST)
 
 e2e-test: $(MAIN_BINARY)
-	@python3 e2e/e2e.py run --ppp $(abspath $(BUILD_DIR)/$(MAIN_BINARY))
+	@python3 $(E2E_TEST_DIR)/e2e.py run --ppp $(abspath $(BUILD_DIR)/$(MAIN_BINARY))
 
-e2e-analyze-latest:
-	@python3 e2e/e2e.py analyze --dir "[latest-test-result-dir]"
+e2e-analyze:
+	@python3 $(E2E_TEST_DIR)/e2e.py analyze --dir "[latest-test-result-dir]"
 
-e2e-analyze-latest-vscode:
-	@python3 e2e/e2e.py analyze --dir "[latest-test-result-dir]" --vscode
+e2e-analyze-in-vscode:
+	@python3 $(E2E_TEST_DIR)/e2e.py analyze --dir "[latest-test-result-dir]" --vscode
+
+e2e-test-pdfs:
+	cd $(E2E_TEST_DIR)/pdfs && latexmk *.tex && latexmk -c
 
 # --------------------------------------------------------------------------------------------------
 # Installing.

diff --git a/e2e/README.md b/e2e/README.md
@@ -0,0 +1,14 @@
+# E2E Testing
+
+- Config:
+    - Definition of paths to the result files
+    - Definition of the tests.
+- Result files:
+    - eo: expected output
+    - ao: actual output
+    - diff: diff result
+    - report: metadata
+- Definition Test
+- Definition Test Case:
+
+- Usage:
diff --git a/e2e/config.yml b/e2e/config.yml
@@ -1,30 +1,36 @@
+# In the following we use the following notation to refer to a particular test case:
+#
+# TEST_CASE(<test-slug>, <pdf-path>)
+#
+# - "<test-slug>" stands for the slug of a test (defined in the 'tests' section of this config) and
+#   refers to the test to which the test case belongs
+# - "<pdf-path>" is the path to the PDF file to be processed by the test case
+
 ---
 paths:
-  # The relative path to the directory containing the PDF files to be processed by the E2E tests.
+  # The (relative) path to the directory containing the PDF files to be processed.
   pdfs_dir: pdfs
 
-  # The relative path to the file containing the expected output of the test with slug
-  # {test-slug} when run on '{pdfs_dir}/{pdf_stem}.pdf'.
+  # The (relative) path to the file containing the expected output of the pdftotext++ command
+  # behind TEST_CASE({test_slug}, {pdfs_dir}/{pdf_stem}.pdf).
   expected_output_file: ground_truth/{test_slug}/{pdf_stem}.eo
 
-  # The relative path to the file containing the actual output of the test with slug
-  # {test_slug} run on '{pdfs_dir}/{pdf_stem}.pdf'. The placeholder {e2e_run_date} stands for the
-  # start date (and time) of 'e2e.py run' and is included to separate the files of different runs.
+  # The (relative) path to the file containing the actual output produced by the pdftotext++
+  # command behind TEST_CASE({test_slug}, {pdfs_dir}/{pdf_stem}.pdf).
+  # The placeholder '{e2e_run_date}' stands for the start date and time of 'e2e.py run' and is used
+  # to store the result files of different test runs in separate folders.
   actual_output_file: results/{e2e_run_date}/{test_slug}/{pdf_stem}.ao
 
-  # The relative path to the file containing the diff between the expected and actual output of the
-  # test with slug {test_slug} run on '{pdfs_dir}/{pdf_stem}.pdf'. The placeholder {e2e_run_date}
-  # stands for the start date (and time) of 'e2e.py run' and is included to separate the files of
-  # different runs.
+  # The (relative) path to the file containing the output produced by the 'diff' command used to
+  # compare the expected and actual output of TEST_CASE({test_slug}, {pdfs_dir}/{pdf_stem}.pdf).
   diff_file: results/{e2e_run_date}/{test_slug}/{pdf_stem}.diff
 
-  # The relative path to the file containing some useful metadata about the (completed) test with
-  # slug {test_slug} run on '{pdfs_dir}/{pdf_stem}.pdf', for example: whether or not the test
-  # succeeded. The placeholder {e2e_run_date} stands for the start date (and time) of 'e2e.py run'
-  # and is included to separate the files of different runs.
+  # The (relative) path to the file containing some useful metadata about the (completed)
+  # TEST_CASE({test_slug}, {pdfs_dir}/{pdf_stem}.pdf), for example: whether or not the test case
+  # succeeded, or the path to the processed PDF file.
   report_file: results/{e2e_run_date}/{test_slug}/{pdf_stem}.report
 
 tests:
-  - name: "No options"
+  - name: "Test pdftotext++ without options"
     slug: no-options
     ppp_args: "{pdf}"
diff --git a/e2e/e2e.py b/e2e/e2e.py
@@ -12,14 +12,14 @@
 
 # ==================================================================================================
 
-DESCRIPTION: str = "A tool for running & analyzing the end-to-end tests of pdftotext++."
+DESCRIPTION: str = "A tool for running & analyzing the E2E tests of pdftotext++."
 
 USAGE: str = """e2e.py <command> [<options>]
 
 where <command> is one of:
 
-  run        runs the end-to-end tests of pdftotext++
-  analyze    analyzes E2E test reports
+  run        runs the E2E tests of pdftotext++
+  analyze    analyzes E2E test reports produced during previous test runs
 
 Type \"e2e.py <command> --help\" to read about the options of a specific subcommand.
 """
@@ -29,31 +29,30 @@
 # The default path to the pdftotext++ executable.
 DEFAULT_PPP_PATH: Path = Path("/usr/bin/pdftotext++")
 
-# The path to the default config file (e.g., specifying the tests to run).
+# The default path to the config file (e.g., specifying the tests to run).
 DEFAULT_CONFIG_FILE_PATH: Path = Path(__file__).parent / "config.yml"
 
-# The default path to the directory to scan for report files for analyzing.
+# The default path to the directory to scan for report files.
 DEFAULT_REPORT_DIR_PATH: str = E2eAnalyzer.LATEST_TEST_RESULT_DIR
 
-# The default file name mask to be used to match the report files to analyze.
-DEFAULT_REPORT_FILE_MASK: Path = "*.report"
+# The default file name mask to be used to match report files.
+DEFAULT_REPORT_FILE_NAME_MASK: Path = "*.report"
 
 # ==================================================================================================
 
 
 class E2ECommandLineInterface:
     """
     A class for reading and processing the command line arguments of the different subcommands
-    available for running and analyzing the end-to-end tests of pdftotext++.
+    available for running and analyzing the E2E tests of pdftotext++.
     """
 
     def __init__(self) -> None:
         """
-        Reads and processes the command line arguments of the specified subcommand and invokes the
-        method of this class which is responsible for reading the subcommad-specific arguments.
+        Reads and processes the specified subcommand from the command line and invokes the method
+        which is responsible for reading and processing the arguments of that subcommand.
         """
 
-        # Create a command line arguments parser.
         parser = argparse.ArgumentParser(
             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
             description=f"{Style.BOLD}{DESCRIPTION}{Style.NONE}",
@@ -63,29 +62,25 @@ def __init__(self) -> None:
         # Add an argument for specifying the subcommand to run.
         parser.add_argument("command", help="the command to run")
 
-        # Parse the subcommand.
         args = parser.parse_args(sys.argv[1:2])
 
-        # Abort if the specified subcommand does not exist.
         if not hasattr(self, args.command):
             sprintln(f"unrecognized command \"{args.command}\"", Style.ERROR)
             sprintln("type \"e2e.py --help\" to get usage info.")
             exit(1)
 
-        # Invoke the method with the same name as the subcommand.
         getattr(self, args.command)()
 
     # ==============================================================================================
 
     def run(self) -> None:
         """
-        Reads and processes the command line arguments specific to the subcommand "run".
+        Reads and processes the arguments of the subcommand "run".
         """
 
-        # Create a command line arguments parser.
         parser = argparse.ArgumentParser(
             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-            description=f"{Style.BOLD}Runs the end-to-end tests of pdftotext++.{Style.NONE}",
+            description=f"{Style.BOLD}Runs the E2E tests of pdftotext++.{Style.NONE}",
             usage="e2e.py run [<options>]"
         )
 
@@ -99,7 +94,7 @@ def run(self) -> None:
             help="set the absolute path to the pdftotext++ executable"
         )
 
-        # Add an argument for specifying the path to the config file specifying the E2E tests.
+        # Add an argument for specifying the path to the config file (specifying the E2E tests).
         parser.add_argument(
             "--config",
             metavar="<path>",
@@ -109,21 +104,20 @@ def run(self) -> None:
             help="set the absolute path to the config file in which the tests to run are specified"
         )
 
-        # Parse the command line arguments.
         args = parser.parse_args(sys.argv[2:])
 
-        # Run the end-to-end tests.
         E2eRunner(ppp_path=args.ppp, config_file_path=args.config).run()
 
     def analyze(self) -> None:
         """
-        Analyzes the E2E test reports.
+        Reads and processes the arguments of the subcommand "analyze".
         """
 
-        # Create a command line arguments parser.
         parser = argparse.ArgumentParser(
             formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-            description=f"{Style.BOLD}Analyzes E2E test reports.{Style.NONE}",
+            description=f"{Style.BOLD}"
+                        "Analyzes E2E test reports produced during previous test runs."
+                        f"{Style.NONE}",
             usage="e2e.py analyze [<options>]"
         )
 
@@ -133,33 +127,30 @@ def analyze(self) -> None:
             metavar="<str>",
             required=False,
             default=DEFAULT_REPORT_DIR_PATH,
-            help="set the path to the directory to scan for report files; the keyword "
-                 f"'{E2eAnalyzer.LATEST_TEST_RESULT_DIR}' is replaced by the path to the "
-                 "directory containing all report files produced during the latest execution of "
-                 "the 'run' subcommand."
+            help="set the path to the directory to scan for report files; use the keyword "
+                 f"'{E2eAnalyzer.LATEST_TEST_RESULT_DIR}' to select the directory containing the "
+                 "report files produced during the *latest* test run."
         )
 
-        # Add an argument for specifying the mask to be used to match report files.
+        # Add an argument for specifying the file name mask to be used to match the report files.
         parser.add_argument(
             "--mask",
             metavar="<str>",
             required=False,
-            default=DEFAULT_REPORT_FILE_MASK,
+            default=DEFAULT_REPORT_FILE_NAME_MASK,
             help="set the file name mask to be used to match the report files to analyze"
         )
 
         # Add an argument for selecting the "VS code" mode.
         parser.add_argument(
             "--vscode",
             action="store_true",
-            help="selects a mode that opens the expected and actual output of each failed test "
-                 "case in VS code side-by-side, with the differences highlighted in color"
+            help="select a mode that opens the expected and actual output of each failed test case "
+                 "in Visual Studio Code side-by-side, with the differences highlighted in color"
         )
 
-        # Parse the command line arguments.
         args = parser.parse_args(sys.argv[2:])
 
-        # Run the analyzer.
         E2eAnalyzer(
             dir_path=args.dir,
             report_file_name_mask=args.mask,