Add support for DiffTrace

DiffTrace would be the cross-platform data structure for DiffTest.
This allows the difftest to be dumped online and restored offline.

.github/workflows/main.yml

@@ -108,3 +108,14 @@ jobs:
             ./build/emu -e 0 -i ./ready-to-run/microbench.bin --diff ./ready-to-run/riscv64-nemu-interpreter-so -C 10000 --dump-coverage
             make -C difftest coverage
             ls -lh *.coverage
+
+      - name: Difftest with DiffTrace
+        run: |
+            cd $GITHUB_WORKSPACE/../xs-env
+            source ./env.sh
+            cd $GITHUB_WORKSPACE/../xs-env/NutShell
+            source ./env.sh
+            make clean
+            make emu
+            ./build/emu -e 0 -i ./ready-to-run/microbench.bin --diff ./ready-to-run/riscv64-nemu-interpreter-so --dump-difftrace microbench
+            ./build/emu -e 0 -i ./ready-to-run/microbench.bin --diff ./ready-to-run/riscv64-nemu-interpreter-so --load-difftrace microbench

src/test/csrc/difftest/difftest.cpp

@@ -15,6 +15,7 @@
 ***************************************************************************************/
 
 #include "difftest.h"
+#include "difftrace.h"
 #include "goldenmem.h"
 #include "ram.h"
 #include "flash.h"
@@ -56,6 +57,12 @@ int difftest_step() {
   return 0;
 }
 
+void difftest_trace() {
+  for (int i = 0; i < NUM_CORES; i++) {
+    difftest[i]->trace();
+  }
+}
+
 void difftest_finish() {
   for (int i = 0; i < NUM_CORES; i++) {
     delete difftest[i];
@@ -79,6 +86,7 @@ Difftest::Difftest(int coreid) : id(coreid) {
 
 Difftest::~Difftest() {
   delete state;
+  delete difftrace;
   if (proxy) {
     delete proxy;
   }

src/test/csrc/difftest/difftest.h

@@ -19,6 +19,7 @@
 
 #include <vector>
 #include "common.h"
+#include "difftrace.h"
 #include "emu.h"
 #include "refproxy.h"
 #include "golden.h"
@@ -202,6 +203,17 @@ class Difftest {
     return dut.trap.code;
   }
   void display();
+  void set_trace(const char *name, bool is_read) {
+    difftrace = new DiffTrace(name, is_read);
+  }
+  void trace() {
+    if (difftrace) {
+      if (difftrace->is_read)
+        difftrace->read_next(&dut);
+      else
+        difftrace->append(&dut);
+    }
+  }
 
   // Difftest public APIs for dut: called from DPI-C functions (or testbench)
   // These functions generally do nothing but copy the information to core_state.
@@ -236,6 +248,8 @@ class Difftest {
   }
 
 protected:
+  DiffTrace *difftrace = nullptr;
+
 #if defined(CPU_NUTSHELL) || defined(CPU_ROCKET_CHIP)
   const uint64_t firstCommit_limit = 1000;
   const uint64_t stuck_limit = 500;
@@ -325,6 +339,7 @@ int difftest_init();
 int difftest_step();
 int difftest_state();
 void difftest_finish();
+void difftest_trace();
 int init_nemuproxy(size_t);
 
 #endif

src/test/csrc/difftest/difftrace.cpp

@@ -0,0 +1,82 @@
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "difftrace.h"
+
+DiffTrace::DiffTrace(const char *_trace_name, bool is_read, uint64_t _buffer_size)
+  : is_read(is_read) {
+  if (!is_read) {
+    buffer_size = _buffer_size;
+    buffer = (DiffTestState *)calloc(buffer_size, sizeof(DiffTestState));
+  }
+  if (strlen(trace_name) > 31) {
+    printf("Length of trace_name %s is more than 31 characters.\n", trace_name);
+    printf("Please use a shorter name.\n");
+    exit(0);
+  }
+  strcpy(trace_name, _trace_name);
+}
+
+bool DiffTrace::append(const DiffTestState *trace) {
+  memcpy(buffer + buffer_count, trace, sizeof(DiffTestState));
+  buffer_count++;
+  if (buffer_count == buffer_size) {
+    return trace_file_next();
+  }
+  return 0;
+}
+
+bool DiffTrace::read_next(DiffTestState *trace) {
+  if (!buffer || buffer_count == buffer_size) {
+    trace_file_next();
+  }
+  memcpy(trace, buffer + buffer_count, sizeof(DiffTestState));
+  buffer_count++;
+  // printf("%lu...\n", buffer_count);
+  return 0;
+}
+
+
+bool DiffTrace::trace_file_next() {
+  static uint64_t trace_index = 0;
+  static FILE *file = nullptr;
+  if (file) {
+    fclose(file);
+  }
+  char filename[128];
+  char *noop_home = getenv("NOOP_HOME");
+  snprintf(filename, 128, "%s/%s", noop_home, trace_name);
+  mkdir(filename, 0755);
+  const char *prefix = "bin";
+  snprintf(filename, 128, "%s/%s/%lu.%s", noop_home, trace_name, trace_index, prefix);
+  if (is_read) {
+    FILE *file = fopen(filename, "rb");
+    if (!file) {
+      printf("File %s not found.\n", filename);
+      exit(0);
+    }
+    // check the number of traces
+    fseek(file, 0, SEEK_END);
+    buffer_size = ftell(file) / sizeof(DiffTestState);
+    if (buffer) {
+      free(buffer);
+    }
+    buffer = (DiffTestState *)calloc(buffer_size, sizeof(DiffTestState));
+    // read the binary file
+    Info("Loading %lu traces from %s ...\n", buffer_size, filename);
+    fseek(file, 0, SEEK_SET);
+    uint64_t read_bytes = fread(buffer, sizeof(DiffTestState), buffer_size, file);
+    assert(read_bytes == buffer_size);
+    fclose(file);
+    buffer_count = 0;
+  }
+  else if (buffer_count > 0) {
+    Info("Writing %lu traces to %s ...\n", buffer_count, filename);
+    FILE *file = fopen(filename, "wb");
+    fwrite(buffer, sizeof(DiffTestState), buffer_count, file);
+    fclose(file);
+    buffer_count = 0;
+  }
+  trace_index++;
+  return 0;
+}

src/test/csrc/difftest/difftrace.h

@@ -0,0 +1,31 @@
+#ifndef __DIFFTRACE_H__
+#define __DIFFTRACE_H__
+
+#include "common.h"
+
+class DiffTrace {
+public:
+  char trace_name[32];
+  bool is_read;
+
+  DiffTrace(const char *trace_name, bool is_read, uint64_t buffer_size = 1024 * 1024);
+  ~DiffTrace() {
+    if (!is_read) {
+      trace_file_next();
+    }
+    if (buffer) {
+      free(buffer);
+    }
+  }
+  bool append(const DiffTestState *trace);
+  bool read_next(DiffTestState *trace);
+
+private:
+  uint64_t buffer_size;
+  uint64_t buffer_count = 0;
+  DiffTestState *buffer = nullptr;
+
+  bool trace_file_next();
+};
+
+#endif

src/test/csrc/verilator/emu.cpp

@@ -105,6 +105,8 @@ static inline void print_help(const char *file) {
 #if VM_COVERAGE == 1
   printf("      --dump-coverage        enable coverage dump\n");
 #endif // VM_COVERAGE
+  printf("      --load-difftrace=NAME  load from trace NAME\n");
+  printf("      --dump-difftrace=NAME  dump to trace NAME\n");
   printf("  -h, --help                 print program help info\n");
   printf("\n");
 }
@@ -133,6 +135,8 @@ inline EmuArgs parse_args(int argc, const char *argv[]) {
     { "dump-coverage",     0, NULL,  0  },
     { "dump-ref-trace",    0, NULL,  0  },
     { "dump-commit-trace", 0, NULL,  0  },
+    { "load-difftrace",    1, NULL,  0  },
+    { "dump-difftrace",    1, NULL,  0  },
     { "seed",              1, NULL, 's' },
     { "max-cycles",        1, NULL, 'C' },
     { "fork-interval",     1, NULL, 'X' },
@@ -198,6 +202,8 @@ inline EmuArgs parse_args(int argc, const char *argv[]) {
             continue;
           case 15: args.enable_ref_trace = true; continue;
           case 16: args.enable_commit_trace = true; continue;
+          case 17: args.trace_name = optarg; args.trace_is_read = true; continue;
+          case 18: args.trace_name = optarg; args.trace_is_read = false; continue;
         }
         // fall through
       default:
@@ -353,6 +359,13 @@ Emulator::Emulator(int argc, const char *argv[]):
 #ifndef CONFIG_NO_DIFFTEST
   // init difftest
   difftest_init();
+
+  // init difftest traces
+  if (args.trace_name) {
+    for (int i = 0; i < NUM_CORES; i++) {
+      difftest[i]->set_trace(args.trace_name, args.trace_is_read);
+    }
+  }
 #endif // CONFIG_NO_DIFFTEST
 
   init_device();
@@ -480,6 +493,9 @@ Emulator::~Emulator() {
 }
 
 inline void Emulator::reset_ncycles(size_t cycles) {
+  if (args.trace_name && args.trace_is_read) {
+    return;
+  }
   for(int i = 0; i < cycles; i++) {
     dut_ptr->reset = 1;
 #ifdef COVERAGE_PORT_RESET
@@ -503,6 +519,10 @@ inline void Emulator::reset_ncycles(size_t cycles) {
 }
 
 inline void Emulator::single_cycle() {
+  if (args.trace_name && args.trace_is_read) {
+    goto end_single_cycle;
+  }
+
   dut_ptr->clock = 0;
 #ifdef COVERAGE_PORT_CLOCK
   dut_ptr->coverage_clock = dut_ptr->clock;
@@ -546,6 +566,11 @@ inline void Emulator::single_cycle() {
   }
 
 end_single_cycle:
+#ifndef CONFIG_NO_DIFFTEST
+  if (args.trace_name) {
+    difftest_trace();
+  }
+#endif // CONFIG_NO_DIFFTEST
   cycles ++;
 }
 

src/test/csrc/verilator/emu.h

@@ -60,6 +60,7 @@ struct EmuArgs {
   const char *ram_size = nullptr;
   const char *flash_bin = nullptr;
   const char *select_db = nullptr;
+  const char *trace_name = nullptr;
   bool enable_waveform = false;
   bool enable_ref_trace = false;
   bool enable_commit_trace = false;
@@ -70,6 +71,7 @@ struct EmuArgs {
   bool enable_jtag = false;
   bool enable_runahead = false;
   bool dump_db = false;
+  bool trace_is_read = true;
   bool dump_coverage = false;
 };