[ci] Add dockerfile and GitHub workflow for the Quidditch toolchain

Since IREE is a large project making use of a lot of `libc` functions, we have high requirements from our toolchain. Establishing a toolchain to use is therefore important and making it have all the right defaults for snitch very convenient.
opencompl · May 10, 2024 · 913d94f · 913d94f
1 parent 55c627f
commit 913d94f
Show file tree

Hide file tree

Showing 8 changed files with 424 additions and 0 deletions.
diff --git a/.github/workflows/toolchain.yaml b/.github/workflows/toolchain.yaml
@@ -0,0 +1,49 @@
+name: Build Docker toolchain image
+on:
+  pull_request:
+    paths:
+      - runtime/toolchain/**
+      - .github/workflows/toolchain.yaml
+  push:
+    branches: [ "main" ]
+    paths:
+      - runtime/toolchain/**
+      - .github/workflows/toolchain.yaml
+  workflow_dispatch:
+
+
+jobs:
+  build-docker:
+    runs-on: ubuntu-22.04
+    steps:
+      # Free up disk space on Github-hosted runner
+      - name: Disk usage
+        run: df -h
+      - uses: jlumbroso/[email protected]
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+
+      - name: Disk usage after freeing up space
+        run: df -h
+      # Actually build the Docker container
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      - name: GHCR Log-in
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/[email protected]
+        with:
+          context: ${{github.workspace}}/runtime/
+          file: ${{github.workspace}}/runtime/toolchain/Dockerfile
+          push: true
+          tags: ghcr.io/opencompl/quidditch/toolchain:${{ github.head_ref || github.ref_name }}
diff --git a/runtime/requirements.txt b/runtime/requirements.txt
@@ -0,0 +1,6 @@
+hjson==3.1.*
+jsonref==1.1.*
+mako==1.3.*
+jsonschema==4.21.*
+tabulate==0.9.*
+PyYAML==6.0.*
diff --git a/runtime/toolchain/Dockerfile b/runtime/toolchain/Dockerfile
@@ -0,0 +1,114 @@
+
+# Global args.
+ARG LLVM_VERSION=18
+ARG PICOLIBC_VERSION=1.8.6
+ARG SNITCH_CLUSTER_SHA=e02cc9e3f24b92d4607455d5345caba3eb6273b2
+ARG VERILATOR_RELEASE=v4.228
+ARG BENDER_RELEASE=0.27.1
+ARG INSTALL_DIR=/opt
+
+FROM alpine:3.18 as toolchain-build
+
+# Args needed by a build stage need to redeclared.
+ARG LLVM_VERSION
+ARG PICOLIBC_VERSION
+ARG INSTALL_DIR
+
+RUN apk update && apk add --no-cache cmake git musl-dev clang clang-dev ccache  \
+    python3 ninja py3-pip meson
+
+ENV CCACHE_DIR=/ccache
+
+WORKDIR $INSTALL_DIR
+
+RUN git clone --depth 1  https://github.com/llvm/llvm-project/ -b release/$LLVM_VERSION.x
+
+COPY toolchain/phase1.cmake /root/
+
+RUN mkdir llvm-project/llvm/build && cd llvm-project/llvm/build && \
+    cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR/quidditch-toolchain \
+     -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+     -DCMAKE_C_COMPILER_LAUNCHER=ccache  \
+     -DLLVM_BUILD_STATIC=ON  \
+     -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+     -C /root/phase1.cmake && ninja install
+
+RUN git clone --depth 1 https://github.com/picolibc/picolibc -b $PICOLIBC_VERSION
+
+RUN echo -e "[binaries]\n\
+c = '$INSTALL_DIR/quidditch-toolchain/bin/clang'\n\
+ar = '$INSTALL_DIR/quidditch-toolchain/bin/llvm-ar'\n\
+as = '$INSTALL_DIR/quidditch-toolchain/bin/llvm-as'\n\
+ld = '$INSTALL_DIR/quidditch-toolchain/bin/ld.lld'\n\
+strip = '$INSTALL_DIR/quidditch-toolchain/bin/llvm-strip'\n\
+\n\
+[host_machine]\n\
+system = 'unknown'\n\
+cpu_family = 'riscv'\n\
+cpu = 'riscv'\n\
+endian = 'little'\n\
+" > /cross.txt
+
+RUN mkdir picolibc/build && cd picolibc/build &&  \
+    meson ..  \
+    -Dincludedir=include \
+    -Dlibdir=lib  \
+    --cross-file /cross.txt  \
+    -Dpicocrt=false  \
+    -Dpicolib=false  \
+    -Dposix-console=true  \
+    -Dprefix=$INSTALL_DIR/quidditch-toolchain  \
+    -Dspecsdir=none && ninja install
+
+COPY toolchain/phase2.cmake /root/
+
+RUN cd llvm-project/llvm/build && rm -rf * && \
+    cmake .. -GNinja -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR/quidditch-toolchain \
+         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+         -DCMAKE_C_COMPILER_LAUNCHER=ccache  \
+         -DLLVM_BUILD_STATIC=ON  \
+         -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \
+         -C  /root/phase2.cmake && \
+        ninja install/strip
+
+FROM alpine:3.18 as verilator-build
+
+ARG SNITCH_CLUSTER_SHA
+ARG VERILATOR_RELEASE
+ARG BENDER_RELEASE
+ARG INSTALL_DIR
+
+ENV CCACHE_DIR=/ccache
+
+WORKDIR $INSTALL_DIR
+
+RUN apk update && apk add --no-cache py3-pip git ccache cargo autoconf flex-dev bison make gcc g++ coreutils dtc patch
+
+RUN cargo install bender --locked --version $BENDER_RELEASE
+ENV PATH=$PATH:/root/.cargo/bin
+COPY requirements.txt /root/
+RUN pip install --break-system-packages -r /root/requirements.txt
+RUN git clone https://github.com/pulp-platform/snitch_cluster && \
+    cd snitch_cluster && git checkout $SNITCH_CLUSTER_SHA && git submodule update --init --recursive
+
+# snitch_cluster repo does not yet support Verilator 5, see: https://github.com/pulp-platform/snitch_cluster/pull/76.
+ENV VERILATOR_ROOT=$INSTALL_DIR/verilator
+RUN git clone --depth 1 https://github.com/verilator/verilator -b $VERILATOR_RELEASE
+RUN cd verilator && autoconf && ./configure && make -j$(nproc)
+ENV VLT=$VERILATOR_ROOT/bin/verilator
+COPY toolchain/snRuntime-01.patch /root/
+COPY toolchain/static-verilator-02.patch /root/
+RUN cd ./snitch_cluster && git apply /root/snRuntime-01.patch && git apply /root/static-verilator-02.patch
+RUN cd ./snitch_cluster/target/snitch_cluster && \
+    make bin/snitch_cluster.vlt -j$(nproc)
+
+RUN mkdir -p $INSTALL_DIR/quidditch-toolchain/bin/
+RUN cp ./snitch_cluster/target/snitch_cluster/bin/snitch_cluster.vlt $INSTALL_DIR/quidditch-toolchain/bin/
+RUN strip $INSTALL_DIR/quidditch-toolchain/bin/snitch_cluster.vlt
+
+FROM alpine:3.18
+
+ARG INSTALL_DIR
+
+COPY --from=toolchain-build $INSTALL_DIR/quidditch-toolchain $INSTALL_DIR/quidditch-toolchain
+COPY --from=verilator-build $INSTALL_DIR/quidditch-toolchain/bin/snitch_cluster.vlt $INSTALL_DIR/quidditch-toolchain/bin
diff --git a/runtime/toolchain/README.md b/runtime/toolchain/README.md
@@ -0,0 +1,60 @@
+# Quidditch Toolchain
+
+This repo contains the `Dockerfile`, cmake build files and transient patches used to build the toolchain for the
+quidditch runtime.
+
+The goals of the toolchain are:
+
+* Minimal: The image is currently "only" 800MB.
+* Easy to maintain: Only upstream components are used, no forks.
+* Modern: The toolchain support C and C++ and uses a full LLVM 18 toolchain with `picolibc` as C standard library.
+
+Why not https://github.com/pulp-platform/llvm-project/?
+The pulp toolchain is based on LLVM 12 and a fork of LLVM, therefore not easily maintainable.
+Furthermore, the toolchain has generally not been used for larger applications requiring a more complete `libc` and
+therefore does not have out-of-the-box support for things like `malloc`.
+**Note that this toolchain does not support any of the intrinsics or mnemonics that the pulp toolchain does and never
+will**
+
+## Installation
+
+### Installing locally
+
+The toolchain is released as an alpine docker image and built as fully static binaries that are capable of running on
+any linux distro.
+
+To copy the toolchain run:
+
+```shell
+docker run --rm ghcr.io/opencompl/Quidditch/toolchain:main tar -cC /opt/quidditch-toolchain .\
+ | tar -xC $INSTALL_DIR/quidditch-toolchain
+```
+
+### Using in Docker
+
+Integrating into another docker image can be done
+using [`COPY --from`](https://docs.docker.com/reference/dockerfile/#copy---from)
+
+Example:
+
+```dockerfile
+COPY --from=ghcr.io/opencompl/Quidditch/toolchain:main /opt/quidditch-toolchain $INSTALL_DIR/quidditch-toolchain
+```
+
+## Using in CMake
+
+The toolchain ships with a so-called toolchain file that is used to tell cmake about the cross compilation
+environment.
+The toolchain file is located at `<root-dir>/ToolchainFile.cmake`.
+When building with cmake, add `-DCMAKE_TOOLCHAIN_FILE=<root-dir>/ToolchainFile.cmake` to your `cmake` command line to
+start using the toolchain.
+
+## Building the Docker
+
+Building the docker image requires running
+
+```shell
+docker build -f toolchain/Dockerfile -t <image-name> .
+```
+
+from within the `<root>/runtime` directory.
diff --git a/runtime/toolchain/phase1.cmake b/runtime/toolchain/phase1.cmake
@@ -0,0 +1,76 @@
+set(CMAKE_C_COMPILER clang CACHE STRING "")
+set(CMAKE_CXX_COMPILER clang++ CACHE STRING "")
+set(LLVM_DEFAULT_TARGET_TRIPLE riscv32-unknown-unknown-elf CACHE STRING "")
+set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "")
+set(LLVM_INCLUDE_TESTS OFF CACHE BOOL "")
+set(LLVM_INCLUDE_BENCHMARKS OFF CACHE BOOL "")
+set(LLVM_INCLUDE_DOCS ON CACHE BOOL "")
+set(CMAKE_BUILD_TYPE Release CACHE STRING "")
+set(LLVM_ENABLE_PROJECTS clang;lld CACHE STRING "")
+set(LLVM_INSTALL_TOOLCHAIN_ONLY ON CACHE BOOL "")
+set(LLVM_USE_CRT_RELEASE "MT" CACHE STRING "")
+set(LLVM_APPEND_VC_REV OFF CACHE BOOL "")
+set(LLVM_USE_LLD ON CACHE BOOL "")
+set(LLVM_TOOLCHAIN_TOOLS llvm-dis;llvm-ar;llvm-ranlib;llvm-nm;llvm-objcopy;llvm-objdump;llvm-rc;llvm-profdata;llvm-symbolizer;llvm-strip;llvm-cov;llvm-cxxfilt;llvm-size;llvm-undname;llvm-addr2line;llvm-as;llvm-cat;llvm-cxxdump;llvm-dis;llvm-dwp;llvm-mc;llvm-mca;llvm-mt;obj2yaml;yaml2obj;llvm-link;llvm-lto;llvm-lto2;llvm-readobj;llvm-shlib;llvm-split;llvm-strings;sancov;llvm-dwarfdump;llvm-cat;llvm-diff CACHE STRING "")
+set(LLVM_TARGETS_TO_BUILD "RISCV" CACHE STRING "")
+
+set(LIBCLANG_BUILD_STATIC ON CACHE BOOL "")
+set(CLANG_DEFAULT_LINKER lld CACHE STRING "")
+set(CLANG_DEFAULT_OBJCOPY llvm-objcopy CACHE STRING "")
+set(CLANG_DEFAULT_RTLIB compiler-rt CACHE STRING "")
+set(CLANG_DEFAULT_CXX_STDLIB libc++ CACHE STRING "")
+set(CLANG_ENABLE_ARCMT OFF CACHE BOOL "")
+set(CLANG_ENABLE_STATIC_ANALYZER OFF CACHE BOOL "")
+
+file(MAKE_DIRECTORY ${CMAKE_INSTALL_PREFIX}/bin)
+file(WRITE ${CMAKE_INSTALL_PREFIX}/bin/riscv32-unknown-unknown-elf.cfg [=[
+-march=rv32imafdzfh
+-mabi=ilp32d
+-mcmodel=medany
+-static
+-ftls-model=local-exec
+-fno-common
+-mcpu=generic-rv32
+-nostartfiles
+# For libc++ and friends.
+--sysroot=<CFGDIR>/..
+-Wno-unused-command-line-argument
+-ffunction-sections
+-fdata-sections
+-Wl,--gc-sections
+-Wl,-z,norelro
+-fvisibility=hidden
+]=])
+file(WRITE ${CMAKE_INSTALL_PREFIX}/bin/clang++.cfg [=[
+-nostdlib++
+-lc++
+-lc++abi
+]=])
+file(WRITE ${CMAKE_INSTALL_PREFIX}/bin/clang++.cfg [=[
+-nostdlib++
+-lc++
+-lc++abi
+]=])
+file(WRITE ${CMAKE_INSTALL_PREFIX}/ToolchainFile.cmake [=[
+set(QUIDDITCH_TOOLCHAIN_ROOT ${CMAKE_CURRENT_LIST_DIR})
+
+# Without that flag CMake is not able to pass test compilation check
+set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY CACHE INTERNAL "")
+
+set(CMAKE_SYSTEM_NAME               Generic CACHE INTERNAL "")
+set(CMAKE_SYSTEM_PROCESSOR          riscv32 CACHE INTERNAL "")
+
+set(CMAKE_AR ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/llvm-ar${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_ASM_COMPILER ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/clang${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_C_COMPILER ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/clang${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_CXX_COMPILER ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/clang++${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_OBJCOPY ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/llvm-objcopy${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_RANLIB ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/llvm-ranlib${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_SIZE ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/llvm-size${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_STRIP ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/llvm-strip${CMAKE_EXECUTABLE_SUFFIX} CACHE INTERNAL "")
+set(CMAKE_CROSSCOMPILING_EMULATOR ${QUIDDITCH_TOOLCHAIN_ROOT}/bin/snitch_cluster.vlt CACHE INTERNAL "")
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+]=])
diff --git a/runtime/toolchain/phase2.cmake b/runtime/toolchain/phase2.cmake
@@ -0,0 +1,38 @@
+include(${CMAKE_CURRENT_LIST_DIR}/phase1.cmake)
+
+set(BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_C_FLAGS "--config=${CMAKE_INSTALL_PREFIX}/bin/riscv32-unknown-unknown-elf.cfg" CACHE STRING "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_C_FLAGS ${BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_C_FLAGS} CACHE STRING "")
+set(BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_CXX_FLAGS "--config=${CMAKE_INSTALL_PREFIX}/bin/riscv32-unknown-unknown-elf.cfg --config=${CMAKE_INSTALL_PREFIX}/bin/clang++.cfg" CACHE STRING "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_CXX_FLAGS ${BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_CXX_FLAGS} CACHE STRING "")
+set(BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_ASM_FLAGS "--config=${CMAKE_INSTALL_PREFIX}/bin/riscv32-unknown-unknown-elf.cfg" CACHE STRING "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_ASM_FLAGS ${BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_CXX_FLAGS} CACHE STRING "")
+
+set(LLVM_BUILTIN_TARGETS "${LLVM_DEFAULT_TARGET_TRIPLE}" CACHE STRING "")
+set(LLVM_RUNTIME_TARGETS "${LLVM_BUILTIN_TARGETS}" CACHE STRING "")
+set(LLVM_ENABLE_RUNTIMES "compiler-rt;libcxx;libcxxabi" CACHE STRING "")
+set(BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_SYSTEM_NAME Generic CACHE STRING "")
+set(BUILTINS_${LLVM_DEFAULT_TARGET_TRIPLE}_COMPILER_RT_BAREMETAL_BUILD ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_CMAKE_SYSTEM_NAME Generic CACHE STRING "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_COMPILER_RT_BAREMETAL_BUILD ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_BAREMETAL ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_ENABLE_ASSERTIONS OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_ENABLE_SHARED OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_ENABLE_STATIC ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_USE_LLVM_UNWINDER OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_ENABLE_THREADS OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_USE_COMPILER_RT ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXXABI_SILENT_TERMINATE ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ABI_UNSTABLE ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_EXCEPTIONS OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_FILESYSTEM OFF CACHE STRING "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_MONOTONIC_CLOCK OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_RANDOM_DEVICE OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_RTTI OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_SHARED OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_STATIC ON CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_THREADS OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_WIDE_CHARACTERS OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_LOCALIZATION OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_ENABLE_UNICODE OFF CACHE BOOL "")
+set(RUNTIMES_${LLVM_DEFAULT_TARGET_TRIPLE}_LIBCXX_USE_COMPILER_RT ON CACHE BOOL "")