[HW/SW] Cheshire integration - Linux on FPGA

.github/workflows/ci.yml

@@ -28,14 +28,14 @@ jobs:
   tc-llvm:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Recover the submodule commit hash
       id: recover_hash
       run: |
         git submodule status toolchain/riscv-llvm | cut -d' ' -f1
         echo "tc-llvm-hash=`git submodule status toolchain/riscv-llvm | cut -d' ' -f1`" >> $GITHUB_ENV
     - name: Cache the LLVM toolchain
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: tc-llvm-cache
       env:
         cache-name: cache-llvm
@@ -67,14 +67,14 @@ jobs:
   tc-gcc:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Recover the submodule commit hash
       id: recover_hash
       run: |
         git submodule status toolchain/riscv-gnu-toolchain | cut -d' ' -f1
         echo "tc-gcc-hash=`git submodule status toolchain/riscv-gnu-toolchain | cut -d' ' -f1`" >> $GITHUB_ENV
     - name: Cache the GCC toolchain
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: tc-gcc-cache
       env:
         cache-name: cache-gcc
@@ -102,14 +102,14 @@ jobs:
   tc-isa-sim:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Recover the submodule commit hash
       id: recover_hash
       run: |
         git submodule status toolchain/riscv-isa-sim | cut -d' ' -f1
         echo "tc-isa-sim-hash=`git submodule status toolchain/riscv-isa-sim | cut -d' ' -f1`" >> $GITHUB_ENV
     - name: Cache Spike
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: tc-isa-sim-cache
       env:
         cache-name: cache-spike
@@ -137,14 +137,14 @@ jobs:
   tc-verilator:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Recover the submodule commit hash
       id: recover_hash
       run: |
         git submodule status toolchain/verilator | cut -d' ' -f1
         echo "tc-verilator-hash=`git submodule status toolchain/verilator | cut -d' ' -f1`" >> $GITHUB_ENV
     - name: Cache Verilator
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: tc-verilator-cache
       env:
         cache-name: cache-verilator
@@ -182,7 +182,7 @@ jobs:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: tc-llvm
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: 3.6
@@ -214,7 +214,7 @@ jobs:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: ["tc-llvm", "tc-gcc", "tc-isa-sim"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Download Spike
       run: |
         git submodule update --init --recursive -- toolchain/riscv-isa-sim
@@ -253,7 +253,7 @@ jobs:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: ["tc-verilator", "tc-isa-sim"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Download Spike
       run: |
         git submodule update --init --recursive -- toolchain/riscv-isa-sim
@@ -304,7 +304,7 @@ jobs:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: ["compile-ara", "compile-apps"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get Spike artifacts
       uses: actions/download-artifact@v4
       with:
@@ -337,7 +337,7 @@ jobs:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: ["compile-ara", "compile-riscv-tests"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get Spike artifacts
       uses: actions/download-artifact@v4
       with:
@@ -362,7 +362,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: ["tc-isa-sim", "compile-riscv-tests"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get Spike artifacts
       uses: actions/download-artifact@v4
       with:
@@ -400,7 +400,7 @@ jobs:
   check-license:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: 3.6
@@ -413,7 +413,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: ['tc-llvm']
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: 3.6
@@ -438,7 +438,7 @@ jobs:
   check-trailing-whitespaces:
     runs-on: ubuntu-20.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
     - name: Determine base commit
@@ -471,7 +471,7 @@ jobs:
         ara_config: [2_lanes, 4_lanes, 8_lanes, 16_lanes]
     needs: ["compile-ara", "compile-apps"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: 3.6
@@ -519,13 +519,13 @@ jobs:
     runs-on: ubuntu-20.04
     needs: benchmark
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: 3.6
     - name: Install Python requirements
       run: pip install -r python-requirements.txt
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Get [f]dotproduct results (2 lanes)
       uses: actions/download-artifact@v4
       with:
@@ -676,7 +676,7 @@ jobs:
     if: always()
     needs: ["simulate", "riscv-tests-spike", "riscv-tests-simv"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Delete artifacts
         uses: geekyeggo/delete-artifact@v5
         with:
@@ -696,7 +696,7 @@ jobs:
     if: always()
     needs: ["simulate", "riscv-tests-spike", "riscv-tests-simv"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Delete artifacts
         uses: geekyeggo/delete-artifact@v5
         with:

.gitmodules

@@ -17,3 +17,6 @@
 	path = toolchain/riscv-llvm
 	url = https://github.com/llvm/llvm-project.git
 	ignore = dirty
+[submodule "cheshire/sw/cva6-sdk"]
+	path = cheshire/sw/cva6-sdk
+	url = [email protected]:moimfeld/cva6-sdk.git

CHANGELOG.md

@@ -47,6 +47,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
  - Add support for vrgather/vcompress
  - Add parametrized support for 8-alt, 8, 16-alt floating-point
  - Add lavaMD `app`, benchmark, and performance plot
+ - Add Cheshire bare-metal FPGA flow for vcu128 and vcu118
+ - Add cva6-sdk submodule
+ - Add Cheshire Linux FPGA flow for vcu128 and vcu118
 
 ### Changed
 
@@ -81,6 +84,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
  - The MASKU always receives balanced payloads from the lanes
  - Remove FPU support for opqueues that do not need it
  - Pre-calculate timing-critical addresses before addrgen stage
+ - Update all GitHub Actions for CI
+ - Update READMEs with FPGA implementation instructions
 
 ## 3.0.0 - 2023-09-08
 

README.md

@@ -211,6 +211,12 @@ Currently, the following kernels support automatic VCD dumping: `fmatmul`, `fcon
 
 We also provide Synopsys Spyglass linting scripts in the hardware/spyglass. Run make lint in the hardware folder, with a specific MemPool configuration, to run the tests associated with the lint_rtl target.
 
+## FPGA implementation and Linux flow
+
+Ara supports Cheshire's FPGA flow and can be currently implemented on VCU128 and VCU118 in bare-metal and with Linux. The tested configuration is with 2 lanes.
+
+For information about the FPGA bare-metal and Linux flows, please refer to `cheshire/README.md`.
+
 ## Publications
 
 If you want to use Ara, you can cite us:

apps/Makefile

@@ -37,6 +37,13 @@ CVA6_BINARIES := $(addprefix bin/, $(cva6_tests))
 ARA_EXTENSIONS := rv64uv
 ARA_BINARIES := $(addprefix bin/, $(ara_tests))
 
+# Suffix for binaries
+ifeq ($(LINUX),1)
+BIN_SUFFIX := -linux
+else
+BIN_SUFFIX :=
+endif
+
 # FFT requires special treatment because of its header files
 ifeq ($(ENV_DEFINES),)
 bin/fft:  ENV_DEFINES += -DFFT_SAMPLES=$(subst ",,$(firstword $(def_args_fft)))
@@ -98,9 +105,9 @@ endef
 $(foreach app,$(APPS),$(eval $(call app_compile_template_spike,$(app))))
 
 define app_compile_template
-bin/$1: $1/data.S.o $(addsuffix .o, $(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) linker_script
+bin/$1$(BIN_SUFFIX): $1/data.S.o $(addsuffix .o, $(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC) -Iinclude $(RISCV_CCFLAGS) -o $$@ $$(addsuffix .o, $$(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) $$(RISCV_LDFLAGS) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC) -Iinclude $(RISCV_CCFLAGS) -o $$@ $$(addsuffix .o, $$(shell find $(1) -name "*.c" -o -name "*.S")) $(RUNTIME_LLVM) $$(RISCV_LDFLAGS)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef
@@ -114,7 +121,7 @@ TESTS_$(1) := $(addprefix bin/, $($(addsuffix _ara_tests, $1)))
 
 bin/$(1)-ara-%: $(TESTS_DIR)/$(1)/%.$(2) $(RUNTIME_GCC) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC_GCC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS_GCC) $$(RISCV_LDFLAGS_GCC) -o $$@ $$< $(RUNTIME_GCC) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC_GCC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS_GCC) $$(RISCV_LDFLAGS_GCC) -o $$@ $$< $(RUNTIME_GCC)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef
@@ -124,7 +131,7 @@ TESTS_$(1) := $(addprefix bin/, $($(addsuffix _ara_tests, $1)))
 
 bin/$(1)-ara-%: $(TESTS_DIR)/$(1)/%.$(2) $(RUNTIME_LLVM) linker_script
 	mkdir -p bin/
-	$$(RISCV_CC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS) $$(RISCV_LDFLAGS) -o $$@ $$< $(RUNTIME_LLVM) -T$$(CURDIR)/common/link.ld
+	$$(RISCV_CC) -Iinclude -I$$(TESTS_DIR)/macros/scalar -I$$(TESTS_DIR)/macros/vector $$(RISCV_CCFLAGS) $$(RISCV_LDFLAGS) -o $$@ $$< $(RUNTIME_LLVM)
 	$$(RISCV_OBJDUMP) $$(RISCV_OBJDUMP_FLAGS) -D $$@ > [email protected]
 	$$(RISCV_STRIP) $$@ -S --strip-unneeded
 endef

apps/common/runtime.mk

@@ -44,7 +44,18 @@ RISCV_ARCH    ?= rv$(RISCV_XLEN)gcv
 RISCV_ABI     ?= lp64d
 RISCV_TARGET  ?= riscv$(RISCV_XLEN)-unknown-elf
 
-# Use LLVM
+# Use LLVM for bare-metal RVV and GCC for Linux RVV
+ifeq ($(LINUX),1)
+RISCV_PREFIX  ?= $(ARA_DIR)/cheshire/sw/cva6-sdk/buildroot/output/host/bin/riscv64-buildroot-linux-gnu-
+RISCV_CC      ?= $(RISCV_PREFIX)gcc
+RISCV_CXX     ?= $(RISCV_PREFIX)g++
+RISCV_OBJDUMP ?= $(RISCV_PREFIX)objdump
+RISCV_OBJCOPY ?= $(RISCV_PREFIX)objcopy
+RISCV_AS      ?= $(RISCV_PREFIX)as
+RISCV_AR      ?= $(RISCV_PREFIX)ar
+RISCV_LD      ?= $(RISCV_PREFIX)ld.lld
+RISCV_STRIP   ?= $(RISCV_PREFIX)strip
+else
 RISCV_PREFIX  ?= $(LLVM_INSTALL_DIR)/bin/
 RISCV_CC      ?= $(RISCV_PREFIX)clang
 RISCV_CXX     ?= $(RISCV_PREFIX)clang++
@@ -54,6 +65,7 @@ RISCV_AS      ?= $(RISCV_PREFIX)llvm-as
 RISCV_AR      ?= $(RISCV_PREFIX)llvm-ar
 RISCV_LD      ?= $(RISCV_PREFIX)ld.lld
 RISCV_STRIP   ?= $(RISCV_PREFIX)llvm-strip
+endif
 
 # Use gcc to compile scalar riscv-tests
 RISCV_CC_GCC  ?= $(GCC_INSTALL_DIR)/bin/$(RISCV_TARGET)-gcc
@@ -75,6 +87,9 @@ PYTHON ?= python3
 
 # Defines
 ENV_DEFINES ?=
+ifeq ($(LINUX),1)
+ENV_DEFINES += -DARA_LINUX=1
+endif
 ifeq ($(vcd_dump),1)
 ENV_DEFINES += -DVCD_DUMP=1
 endif
@@ -88,27 +103,38 @@ RISCV_WARNINGS += -Wunused-variable -Wall -Wextra -Wno-unused-command-line-argum
 LLVM_FLAGS     ?= -march=rv64gcv_zfh_zvfh0p1 -menable-experimental-extensions -mabi=$(RISCV_ABI) -mno-relax -fuse-ld=lld
 LLVM_V_FLAGS   ?= -fno-vectorize -mllvm -scalable-vectorization=off -mllvm -riscv-v-vector-bits-min=0 -Xclang -target-feature -Xclang +no-optimized-zero-stride-load
 RISCV_FLAGS    ?= $(LLVM_FLAGS) $(LLVM_V_FLAGS) -mcmodel=medany -I$(CURDIR)/common -std=gnu99 -O3 -ffast-math -fno-common -fno-builtin-printf $(DEFINES) $(RISCV_WARNINGS)
+ifeq ($(LINUX),1)
+RISCV_CCFLAGS  ?= -march=rv64gcv -mabi=$(RISCV_ABI) -I$(CURDIR)/common -O3 $(DEFINES)
+RISCV_LDFLAGS  ?=
+else
 RISCV_CCFLAGS  ?= $(RISCV_FLAGS) -ffunction-sections -fdata-sections
+RISCV_LDFLAGS  ?= -static -nostartfiles -lm -Wl,--gc-sections -T$(CURDIR)/common/link.ld
+endif
 RISCV_CCFLAGS_SPIKE  ?= $(RISCV_FLAGS) $(SPIKE_CCFLAGS) -ffunction-sections -fdata-sections
 RISCV_CXXFLAGS ?= $(RISCV_FLAGS) -ffunction-sections -fdata-sections
-RISCV_LDFLAGS  ?= -static -nostartfiles -lm -Wl,--gc-sections
 RISCV_LDFLAGS_SPIKE  ?= $(RISCV_LDFLAGS) $(SPIKE_LDFLAGS) -Wl,--gc-sections
 
 # GCC Flags
 RISCV_FLAGS_GCC    ?= -mcmodel=medany -march=$(RISCV_ARCH) -mabi=$(RISCV_ABI) -I$(CURDIR)/common -static -std=gnu99 -O3 -ffast-math -fno-common -fno-builtin-printf $(DEFINES) $(RISCV_WARNINGS)
 RISCV_CCFLAGS_GCC  ?= $(RISCV_FLAGS_GCC)
 RISCV_CXXFLAGS_GCC ?= $(RISCV_FLAGS_GCC)
-RISCV_LDFLAGS_GCC  ?= -static -nostartfiles -lm -lgcc $(RISCV_FLAGS_GCC)
+RISCV_LDFLAGS_GCC  ?= -static -nostartfiles -lm -lgcc $(RISCV_FLAGS_GCC) -T$(CURDIR)/common/link.ld
 
 ifeq ($(COMPILER),gcc)
 	RISCV_OBJDUMP_FLAGS ?=
 else
+ifneq ($(LINUX),1)
 	RISCV_OBJDUMP_FLAGS ?= --mattr=v
 endif
+endif
 
 # Compile two different versions of the runtime, since we cannot link code compiled with two different toolchains
 RUNTIME_GCC   ?= common/crt0-gcc.S.o common/printf-gcc.c.o common/string-gcc.c.o common/serial-gcc.c.o common/util-gcc.c.o
+ifeq ($(LINUX),1)
+RUNTIME_LLVM  ?= common/util-llvm.c.o
+else
 RUNTIME_LLVM  ?= common/crt0-llvm.S.o common/printf-llvm.c.o common/string-llvm.c.o common/serial-llvm.c.o common/util-llvm.c.o
+endif
 RUNTIME_SPIKE ?= $(spike_env_dir)/benchmarks/common/crt.S.o.spike $(spike_env_dir)/benchmarks/common/syscalls.c.o.spike common/util.c.o.spike
 
 .INTERMEDIATE: $(RUNTIME_GCC) $(RUNTIME_LLVM)

apps/common/util.c

@@ -22,6 +22,8 @@
 
 int *__dummy__errno__ptr__;
 
+unsigned long int timer;
+
 // Floating-point similarity check with threshold
 int similarity_check(double a, double b, double threshold) {
   double diff = a - b;

apps/conjugate_gradient/main.c

@@ -25,10 +25,13 @@
 #include "shared_kernel/spmv.h"
 #include "util.h"
 
-#ifndef SPIKE
-#include "printf.h"
-#else
+#ifdef SPIKE
+#include "util.h"
 #include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
+#include "printf.h"
 #endif
 
 #define USE_SPMV 1

apps/cos/main.c

@@ -20,10 +20,18 @@
 #include <string.h>
 
 #include "kernel/cos.h"
-#include "printf.h"
 #include "runtime.h"
+
 #include "util.h"
 
+#ifdef SPIKE
+#include <stdio.h>
+#elif defined ARA_LINUX
+#include <stdio.h>
+#else
+#include "printf.h"
+#endif
+
 extern size_t N_f64;
 extern double angles_f64[] __attribute__((aligned(4 * NR_LANES)));
 extern double results_f64[] __attribute__((aligned(4 * NR_LANES)));