Add parallel support to RISCOF Sentinel DUT. Also print currently-exe…

…cuting tests and verbose printing to dodo.py.

doc/development/testing.md

@@ -541,6 +541,10 @@ It'd be nice to remove the immediate dependency on `make`. Since we don't want
 to _install_ the emulators in the system, it should be in principle possible to
 `cd sail-riscv`, do `opam install . --deps-only`, and then build SAIL manually.
 However, I [couldn't get that to work](https://stackoverflow.com/questions/50942323/opam-install-error-no-solution-found)...
+
+Additionally, the default RISCOF SAIL plugin
+[uses](https://github.com/riscv-software-src/riscof/blob/9fe3597d75757c6c6198e034bbe62e8dceecc755/riscof/Templates/setup/sail_cSim/riscof_sail_cSim.py#L131-L132)
+`make`, and I haven't gotten around to ripping that out yet :).
 ```
 
 As of 11/6/2023, the {ref}`build instructions <riscof:quickstart>`

dodo.py

@@ -346,8 +346,12 @@ def last_testfile(task, values, testfile):  # noqa: D103
 # location. Yes, this is all to support using custom testfiles :).
 @task_params([{"name": "testfile", "short": "t",
                "default": "./tests/riscof/riscof_work/test_list.yaml",
-               "help": "path to alternate test list"}])
-def task_run_riscof(testfile):
+               "help": "path to alternate test list"},
+              {"name": "verbose", "short": "v",
+               "type": bool,
+               "default": False,
+               "help": "print debug messages"}])
+def task_run_riscof(testfile, verbose):
     """run RISCOF tests against Sentinel/Sail, and report results, removes previous run's artifacts"""  # noqa: E501
     riscof_tests = Path("./tests/riscof/")
     riscof_work = riscof_tests / "riscof_work"
@@ -371,18 +375,24 @@ def task_run_riscof(testfile):
     path_tf = Path(testfile)
     if not path_tf.is_absolute():
         path_tf = path_tf.absolute()
-
-    vars = os.environ.copy()
-    vars["PATH"] += os.pathsep + str(riscof_tests.absolute() / "bin")
+    verbose_str = "--verbose debug" if verbose else ""
+
+    # Would be nice if I didn't need this; SAIL plugin takes a PATH in
+    # config.ini, but the default plugin expects it to be an absolute PATH,
+    # when I want relative (and construct full path with absolute()).
+    # Perhaps lightly modify the plugin later.
+    vars_ = os.environ.copy()
+    vars_["PATH"] += os.pathsep + str(riscof_tests.absolute() / "bin")
     return {
         "title": partial(print_title, title="Running RISCOF tests"),
-        "actions": [CmdAction("pdm run riscof run --config=config.ini "
+        "actions": [CmdAction(f"pdm run riscof {verbose_str} run "
+                              "--config=config.ini "
                               "--suite=riscv-arch-test/riscv-test-suite/ "
                               "--env=riscv-arch-test/riscv-test-suite/env "
                               f"--testfile={path_tf} "
                               "--no-browser --no-clean",
                               cwd=riscof_tests,
-                              env=vars)],
+                              env=vars_)],
         "targets": [riscof_work / "report.html"],
         "verbosity": 2,
         "setup": ["_git_init:sail-riscv",

tests/riscof/config.ini

@@ -9,6 +9,8 @@ pluginpath=./sentinel
 ispec=./sentinel/sentinel_isa.yaml
 pspec=./sentinel/sentinel_platform.yaml
 target_run=1
+; Possible to supply on CLI somehow?
+jobs=nproc
 
 [sail_cSim]
 pluginpath=./sail_cSim

tests/riscof/sentinel/riscof_sentinel.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 from sentinel.top import Top
 from amaranth.sim import Simulator
+from multiprocessing import Pool
 
 import riscof.utils as utils
 from riscof.pluginTemplate import pluginTemplate
@@ -157,91 +158,100 @@ def build(self, isa_yaml, platform_yaml):
     def runTests(self, testList):
         # we will iterate over each entry in the testList. Each entry node will
         # be referred to by the variable testname.
-        for testname in testList:
-            top = Top()
-
-            logger.debug('Running Test: {0} on DUT'.format(testname))
-            # for each testname we get all its fields (as described by the
-            # testList format)
-            testentry = testList[testname]
-
-            # we capture the path to the assembly file of this test
-            test = Path(testentry['test_path'])
-
-            # capture the directory where the artifacts of this test will be
-            # dumped/created.
-            test_dir = Path(testentry['work_dir'])
-
-            # name of the elf file after compilation of the test
-            elf = Path(test.stem).with_suffix(".elf")
-
-            # name of the signature file as per requirement of RISCOF. RISCOF
-            # expects the signature to be named as DUT-<dut-name>.signature.
-            # The below variable creates an absolute path of signature file.
-            sig_file = os.path.join(test_dir, self.name[:-1] + ".signature")
-
-            # for each test there are specific compile macros that need to be
-            # enabled. The macros in the testList node only contain the
-            # macros/values. For the gcc toolchain we need to prefix with "-D".
-            # The following does precisely that.
-            compile_macros = ' -D' + " -D".join(testentry['macros'])
-
-            # collect the march string required for the compiler
-            marchstr = testentry['isa'].lower()
-
-            # substitute all variables in the compile command that we created
-            # in the initialize function
-            cmd = self.compile_cmd.format(marchstr, test, elf, compile_macros)
-
-            # just a simple logger statement that shows up on the terminal
-            logger.debug('Compiling test: ' + str(test))
-
-            # the following command spawns a process to run the compile
-            # command. Note here, we are changing the directory for this
-            # command to that pointed by test_dir. If you would like the
-            # artifacts to be dumped else where change the test_dir variable
-            # to the path of your choice.
-            utils.shellCommand(cmd).run(cwd=test_dir)
-
-            bin = Path(elf.stem).with_suffix(".bin")
-            objcopy_run = f'riscv64-unknown-elf-objcopy -O binary {elf} {bin}'
-            utils.shellCommand(objcopy_run).run(cwd=test_dir)
-
-            objdump_run = f'riscv64-unknown-elf-objdump -D {elf} > {test.stem}.disass;'  # noqa: E501
-            utils.shellCommand(objdump_run).run(cwd=test_dir)
-
-            # for debug purposes if you would like stop the DUT plugin after
-            # compilation, you can comment out the lines below and raise a
-            # SystemExit
-            if self.target_run:
-                # TODO: Convert into SoC module (use wishbone.Decoder and
-                # friends)?
-                with open(test_dir / bin, "rb") as fp:
-                    bin_ = bytearray(fp.read())
-
-                mem = Memory(start=0, size=len(bin_))
-
-                for adr in range(0, len(bin_) // 4):
-                    mem[adr] = int.from_bytes(bin_[4 * adr:4 * adr + 4],
-                                            byteorder="little")
-
-                logger.debug("Executing in Amaranth simulator")
-                sim = Simulator(top)
-                sim.add_clock(1 / 12e6)
-                sim.add_testbench(mk_wait_for_host_write(top, sig_file, mem))
-                sim.add_process(mk_read_write_mem(top, mem))
-
-                vcd = test_dir / Path(test.stem).with_suffix(".vcd")
-                gtkw = test_dir / Path(test.stem).with_suffix(".gtkw")
-                with sim.write_vcd(vcd_file=str(vcd),
-                                   gtkw_file=str(gtkw)):
-                    sim.run()
-
-            # post-processing steps can be added here in the template below
-            # postprocess = 'mv {0} temp.sig'.format(sig_file)'
-            # utils.shellCommand(postprocess).run(cwd=test_dir)
+
+        inps = map(lambda tn: (tn, testList[tn]), testList)
+        if self.num_jobs in ("nproc",):
+            njobs = None
+        else:
+            njobs = int(self.num_jobs)
+
+        with Pool(njobs) as p:
+            p.starmap(self._doTest, inps)
 
         # if target runs are not required then we simply exit as this point
         # after running all the makefile targets.
         if not self.target_run:
             raise SystemExit
+
+    def _doTest(self, testname, testentry):
+        top = Top()
+
+        logger.debug('Running Test: {0} on DUT'.format(testname))
+        # for each testname we get all its fields (as described by the
+        # testList format)
+
+        # we capture the path to the assembly file of this test
+        test = Path(testentry['test_path'])
+
+        # capture the directory where the artifacts of this test will be
+        # dumped/created.
+        test_dir = Path(testentry['work_dir'])
+
+        # name of the elf file after compilation of the test
+        elf = Path(test.stem).with_suffix(".elf")
+
+        # name of the signature file as per requirement of RISCOF. RISCOF
+        # expects the signature to be named as DUT-<dut-name>.signature.
+        # The below variable creates an absolute path of signature file.
+        sig_file = os.path.join(test_dir, self.name[:-1] + ".signature")
+
+        # for each test there are specific compile macros that need to be
+        # enabled. The macros in the testList node only contain the
+        # macros/values. For the gcc toolchain we need to prefix with "-D".
+        # The following does precisely that.
+        compile_macros = ' -D' + " -D".join(testentry['macros'])
+
+        # collect the march string required for the compiler
+        marchstr = testentry['isa'].lower()
+
+        # substitute all variables in the compile command that we created
+        # in the initialize function
+        cmd = self.compile_cmd.format(marchstr, test, elf, compile_macros)
+
+        # just a simple logger statement that shows up on the terminal
+        logger.debug('Compiling test: ' + str(test))
+
+        # the following command spawns a process to run the compile
+        # command. Note here, we are changing the directory for this
+        # command to that pointed by test_dir. If you would like the
+        # artifacts to be dumped else where change the test_dir variable
+        # to the path of your choice.
+        utils.shellCommand(cmd).run(cwd=test_dir)
+
+        bin = Path(elf.stem).with_suffix(".bin")
+        objcopy_run = f'riscv64-unknown-elf-objcopy -O binary {elf} {bin}'
+        utils.shellCommand(objcopy_run).run(cwd=test_dir)
+
+        objdump_run = f'riscv64-unknown-elf-objdump -D {elf} > {test.stem}.disass;'  # noqa: E501
+        utils.shellCommand(objdump_run).run(cwd=test_dir)
+
+        # for debug purposes if you would like stop the DUT plugin after
+        # compilation, you can comment out the lines below and raise a
+        # SystemExit
+        if self.target_run:
+            # TODO: Convert into SoC module (use wishbone.Decoder and
+            # friends)?
+            with open(test_dir / bin, "rb") as fp:
+                bin_ = bytearray(fp.read())
+
+            mem = Memory(start=0, size=len(bin_))
+
+            for adr in range(0, len(bin_) // 4):
+                mem[adr] = int.from_bytes(bin_[4 * adr:4 * adr + 4],
+                                        byteorder="little")
+
+            logger.debug("Executing in Amaranth simulator")
+            sim = Simulator(top)
+            sim.add_clock(1 / 12e6)
+            sim.add_testbench(mk_wait_for_host_write(top, sig_file, mem))
+            sim.add_process(mk_read_write_mem(top, mem))
+
+            vcd = test_dir / Path(test.stem).with_suffix(".vcd")
+            gtkw = test_dir / Path(test.stem).with_suffix(".gtkw")
+            with sim.write_vcd(vcd_file=str(vcd),
+                                gtkw_file=str(gtkw)):
+                sim.run()
+
+        # post-processing steps can be added here in the template below
+        # postprocess = 'mv {0} temp.sig'.format(sig_file)'
+        # utils.shellCommand(postprocess).run(cwd=test_dir)