From 291effe69e4fb97e4d6e8f6802045620b679da70 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 5 Sep 2024 13:36:39 -0700 Subject: [PATCH 01/10] Move some compiler-rt build logic If we move this logic out of the runtimes cmake cache, they can be used together when building the toolchain but also independently when building/testing compiler-rt separately from libc++/abi/unwind. Signed-off-by: Brian Cain --- build-toolchain.sh | 1 + hexagon-linux-clangrt.cmake | 14 ++++++++++++++ hexagon-linux-runtimes.cmake | 7 ------- 3 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 hexagon-linux-clangrt.cmake diff --git a/build-toolchain.sh b/build-toolchain.sh index 706f516..bbad0b6 100755 --- a/build-toolchain.sh +++ b/build-toolchain.sh @@ -205,6 +205,7 @@ build_libs() { -DCMAKE_CXX_COMPILER_FORCED:BOOL=ON \ -C ./hexagon-linux-cross.cmake \ -C ./hexagon-linux-runtimes.cmake \ + -C ./hexagon-linux-clangrt.cmake \ -B ./obj_libs \ -S ./llvm-project/runtimes diff --git a/hexagon-linux-clangrt.cmake b/hexagon-linux-clangrt.cmake new file mode 100644 index 0000000..baf1493 --- /dev/null +++ b/hexagon-linux-clangrt.cmake @@ -0,0 +1,14 @@ + +set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "") +set(LLVM_DEFAULT_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "") + +set(COMPILER_RT_USE_LLVM_UNWINDER ON CACHE BOOL "") + +# Some build failures here, including the inline asm in +# `compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h`, so +# we can just disable these for now: +set(COMPILER_RT_BUILD_BUILTINS OFF CACHE BOOL "") +set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "") +set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "") +set(COMPILER_RT_BUILD_MEMPROF OFF CACHE BOOL "") + diff --git a/hexagon-linux-runtimes.cmake b/hexagon-linux-runtimes.cmake index d1c5b0d..670968e 100644 --- a/hexagon-linux-runtimes.cmake +++ b/hexagon-linux-runtimes.cmake @@ -1,5 +1,4 @@ -set(LLVM_DEFAULT_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "") set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "") set(LLVM_ENABLE_RUNTIMES libcxx;libcxxabi;libunwind;compiler-rt CACHE STRING "") set(LIBCXX_INCLUDE_BENCHMARKS OFF CACHE BOOL "") @@ -9,14 +8,8 @@ set(LIBCXXABI_INCLUDE_TESTS OFF CACHE BOOL "") set(LIBUNWIND_INCLUDE_TESTS OFF CACHE BOOL "") set(LIBCXX_CXX_ABI libcxxabi CACHE STRING "") set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "") -set(COMPILER_RT_USE_LLVM_UNWINDER ON CACHE BOOL "") set(LIBCXXABI_ENABLE_SHARED ON CACHE BOOL "") -set(COMPILER_RT_BUILD_BUILTINS OFF CACHE BOOL "") -set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "") -set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "") -set(COMPILER_RT_BUILD_MEMPROF OFF CACHE BOOL "") - set(LIBCXX_HAS_ATOMIC_LIB OFF CACHE BOOL "") set(LIBCXXABI_HAS_ATOMIC_LIB OFF CACHE BOOL "") set(LIBUNWIND_HAS_ATOMIC_LIB OFF CACHE BOOL "") From 930b384c4c5c61e52c9cb8afadbebda008b1db5d Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Mon, 7 Oct 2024 10:43:03 -0700 Subject: [PATCH 02/10] Remove atomic overrides These overrides were required historically because of some limitations/bugs in the compiler/linker/builtins and caused the cmake feature detection tests to fail. Now they're no longer needed so we can remove them. Signed-off-by: Brian Cain --- hexagon-linux-runtimes.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/hexagon-linux-runtimes.cmake b/hexagon-linux-runtimes.cmake index 670968e..7f309de 100644 --- a/hexagon-linux-runtimes.cmake +++ b/hexagon-linux-runtimes.cmake @@ -10,9 +10,6 @@ set(LIBCXX_CXX_ABI libcxxabi CACHE STRING "") set(LIBCXXABI_USE_LLVM_UNWINDER ON CACHE BOOL "") set(LIBCXXABI_ENABLE_SHARED ON CACHE BOOL "") -set(LIBCXX_HAS_ATOMIC_LIB OFF CACHE BOOL "") -set(LIBCXXABI_HAS_ATOMIC_LIB OFF CACHE BOOL "") -set(LIBUNWIND_HAS_ATOMIC_LIB OFF CACHE BOOL "") set(LIBCXX_USE_COMPILER_RT ON CACHE BOOL "") set(LIBCXXABI_USE_COMPILER_RT ON CACHE BOOL "") set(LIBUNWIND_USE_COMPILER_RT ON CACHE BOOL "") From e166684acd0003de4dae5d3ead7b04a038b643ed Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Thu, 5 Sep 2024 13:44:58 -0700 Subject: [PATCH 03/10] COMPILER_RT_DEFAULT_ARCH is used in favor of COMPILER_RT_DEFAULT_TARGET_TRIPLE Signed-off-by: Brian Cain --- hexagon-linux-builtins.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hexagon-linux-builtins.cmake b/hexagon-linux-builtins.cmake index d9c9ff2..4c31af8 100644 --- a/hexagon-linux-builtins.cmake +++ b/hexagon-linux-builtins.cmake @@ -2,7 +2,8 @@ set(CMAKE_ASM_FLAGS "-G0 -mlong-calls -fno-pic" CACHE STRING "") set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL "") set(LLVM_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "") -set(COMPILER_RT_DEFAULT_TARGET_TRIPLE hexagon-unknown-linux-musl CACHE STRING "") + +set(COMPILER_RT_DEFAULT_ARCH hexagon CACHE STRING "") set(COMPILER_RT_BUILD_BUILTINS ON CACHE BOOL "") set(COMPILER_RT_BUILD_SANITIZERS OFF CACHE BOOL "") set(COMPILER_RT_BUILD_XRAY OFF CACHE BOOL "") From c593f5909073012003d0c3453894501591e1eb90 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 13 Sep 2024 12:56:36 -0700 Subject: [PATCH 04/10] Build a simple init/initramfs An init that shutsdown immediately makes for a simple startup/shutdown regression test of linux on QEMU. Signed-off-by: Brian Cain --- Dockerfile | 2 ++ build-rootfs.sh | 26 ++++++++++++++++++++++++-- test_init/Makefile | 12 ++++++++++++ test_init/test_init.c | 20 ++++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 test_init/Makefile create mode 100644 test_init/test_init.c diff --git a/Dockerfile b/Dockerfile index f6df253..2ff4116 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,8 @@ ADD get-src-tarballs.sh /root/hexagon-toolchain/get-src-tarballs.sh ADD *.cmake /root/hexagon-toolchain/ RUN cd /root/hexagon-toolchain && ./get-src-tarballs.sh ${PWD} ${TOOLCHAIN_INSTALL}/manifest +ADD test_init/test_init.c test_init/Makefile /root/hexagon-toolchain/test_init/ + ENV IN_CONTAINER 1 ADD build-toolchain.sh /root/hexagon-toolchain/build-toolchain.sh RUN cd /root/hexagon-toolchain && ./build-toolchain.sh ${ARTIFACT_TAG} diff --git a/build-rootfs.sh b/build-rootfs.sh index a732e18..3815e0d 100755 --- a/build-rootfs.sh +++ b/build-rootfs.sh @@ -126,6 +126,24 @@ build_minivm() { install } +build_init() { + cd ${BASE} + mkdir -p obj_init + cd obj_init + make -C ../test_init install \ + DESTDIR=${ROOTFS} \ + CC=hexagon-unknown-linux-musl-clang \ + LD=hexagon-unknown-linux-musl-clang \ + +} + +build_initramfs() { +# ../scripts/gen_initramfs_list.sh ${ROOTFS} | \ +# cpio --create --format newc > initramfs_data.cpio + cd ${ROOTFS} + find . | cpio --create --format newc > ${BASE}/linux/usr/initramfs_data.cpio +} + build_kernel() { cd ${BASE} mkdir -p obj_linux @@ -188,8 +206,8 @@ cp -ra ${HEX_SYSROOT}/usr ${ROOTFS}/ get_src_tarballs build_minivm -build_kernel build_busybox +build_init #build_dropbear #build_cpython @@ -203,10 +221,14 @@ mount -t proc none /proc mount -t sysfs none /sys mount -t debugfs none /sys/kernel/debug -exec /bin/sh +exec /bin/test_init EOF chmod +x ${ROOTFS}/init +build_initramfs +build_kernel + + if [[ ${MAKE_TARBALLS-0} -eq 1 ]]; then tar c -C $(dirname ${ROOT_INSTALL_REL}) $(basename ${ROOT_INSTALL_REL}) | xz -e9T0 > ${RESULTS_DIR}/hexagon_rootfs_${STAMP}.tar.xz # XZ_OPT="-8 --threads=0" tar c ${RESULTS_DIR}/hexagon_rootfs_${STAMP}.tar.xz -C $(dirname ${ROOT_INSTALL_REL}) $(basename ${ROOT_INSTALL_REL}) diff --git a/test_init/Makefile b/test_init/Makefile new file mode 100644 index 0000000..3d78fc4 --- /dev/null +++ b/test_init/Makefile @@ -0,0 +1,12 @@ + +.PHONY: install all + +CFLAGS=-mcpu=hexagonv68 +LDFLAGS=-static -mcpu=hexagonv68 + +all: test_init + +test_init: test_init.o + +install: test_init + install -D $^ $(DESTDIR)/bin/$^ diff --git a/test_init/test_init.c b/test_init/test_init.c new file mode 100644 index 0000000..9971566 --- /dev/null +++ b/test_init/test_init.c @@ -0,0 +1,20 @@ +#include +#include +#include +#include +#include +#include +#include + +int main() { + /* For now, let's immediately shutdown. */ + int ret = reboot(LINUX_REBOOT_CMD_HALT); + if (!ret) { + perror("reboot"); + } + + /* Since we're an init process, we should stay + * running. + */ + while (true) { /* spin */ } +} From d87b926cfc9d7ebb613f2461daaae844a2ab693c Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 4 Oct 2024 08:32:25 -0500 Subject: [PATCH 05/10] Use independent ARGs for Dockerfile Signed-off-by: Brian Cain --- Dockerfile | 5 ++++- cross-win/Dockerfile | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2ff4116..670d1e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,10 @@ RUN apt update && \ apt build-dep -yy --arch-only qemu clang python3 # From env.sh -ARG QEMU_REPO QEMU_REF ARTIFACT_BASE ARTIFACT_TAG +ARG QEMU_REPO +ARG QEMU_REF +ARG ARTIFACT_BASE +ARG ARTIFACT_TAG ENV VER 19.1.0 ENV TOOLCHAIN_INSTALL /usr/local/clang+llvm-${VER}-cross-hexagon-unknown-linux-musl/ diff --git a/cross-win/Dockerfile b/cross-win/Dockerfile index 0387e27..513597d 100644 --- a/cross-win/Dockerfile +++ b/cross-win/Dockerfile @@ -35,7 +35,10 @@ RUN winecfg || true COPY cross-win/wine/* /root/.wine/ # From env.sh -ARG QEMU_REPO QEMU_REF ARTIFACT_BASE ARTIFACT_TAG +ARG QEMU_REPO +ARG QEMU_REF +ARG ARTIFACT_BASE +ARG ARTIFACT_TAG ARG QEMU_INSTALL_DIR=qemu-hexagon-x86_64-win ARG QEMU_CLONE_DIR=/root/qemu From 1a4574147ccaa4a9eac6bc2753191d180d3c4561 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 4 Oct 2024 08:38:39 -0500 Subject: [PATCH 06/10] Update windows lib dependency mingw-w64-x86_64-pcre2-10.41-1-any.pkg.tar.zst is no longer available at that URL. Signed-off-by: Brian Cain --- cross-win/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cross-win/Dockerfile b/cross-win/Dockerfile index 513597d..5280ed4 100644 --- a/cross-win/Dockerfile +++ b/cross-win/Dockerfile @@ -17,7 +17,7 @@ RUN for url in https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-glib2-2.74. https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pixman-0.42.2-1-any.pkg.tar.zst \ https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libpng-1.6.40-1-any.pkg.tar.zst \ https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gettext-0.21.1-2-any.pkg.tar.zst \ - https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pcre2-10.41-1-any.pkg.tar.zst \ + https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pcre2-10.44-1-any.pkg.tar.zst \ https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libiconv-1.17-3-any.pkg.tar.zst; do \ wget $url && tar xvf $(basename $url) || exit 1; \ done && \ From c1e5766a621ad1b093254b81866433b06241e4f6 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 4 Oct 2024 19:25:46 -0500 Subject: [PATCH 07/10] Update to latest QEMU tag Signed-off-by: Brian Cain --- Dockerfile | 3 ++- build-in-container.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 670d1e5..6bc626e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,7 +55,8 @@ RUN apt update && \ # From env.sh ARG QEMU_REPO -ARG QEMU_REF +ARG QEMU_REF=hexagon-sysemu-04-oct-2024 + ARG ARTIFACT_BASE ARG ARTIFACT_TAG diff --git a/build-in-container.sh b/build-in-container.sh index ded3778..4302074 100755 --- a/build-in-container.sh +++ b/build-in-container.sh @@ -14,7 +14,7 @@ fi DOCKER_BUILD_ARGS=" --build-arg ARTIFACT_BASE=/usr/local/hexagon-artifacts --build-arg QEMU_REPO=https://github.com/quic/qemu ---build-arg QEMU_REF=hexagon-sysemu-12-dec-2023 +--build-arg QEMU_REF=hexagon-sysemu-04-oct-2024 --build-arg ARTIFACT_TAG=${ARTIFACT_TAG}" #build From d10fa549ca9d9f4c72076f6f5cea58c1462d2d71 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Fri, 4 Oct 2024 19:25:46 -0500 Subject: [PATCH 08/10] Permit failure to remove `tmp_container` On the first time through, `tmp_container` may not yet exist. Signed-off-by: Brian Cain --- build-in-container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-in-container.sh b/build-in-container.sh index 4302074..00e46c4 100755 --- a/build-in-container.sh +++ b/build-in-container.sh @@ -26,7 +26,7 @@ docker images #extract artifacts for name in hexagon hexagon-win; do - docker rm -f tmp_container + docker rm -f tmp_container || /bin/true docker create --name tmp_container $name:latest docker cp tmp_container:/usr/local/hexagon-artifacts ./hexagon-artifacts docker rm tmp_container From c7b9314a4e4d04db2edb2257627f5d720eab63d8 Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Sat, 5 Oct 2024 05:47:43 -0700 Subject: [PATCH 09/10] Add the LLVM test suite to dockerfile Signed-off-by: Brian Cain --- Dockerfile | 1 + get-src-tarballs.sh | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/Dockerfile b/Dockerfile index 6bc626e..fd814df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,7 @@ ENV MAKE_TARBALLS 1 #ENV CMAKE_URL https://github.com/Kitware/CMake/releases/download/v3.16.6/cmake-3.16.6-Linux-x86_64.tar.gz ENV LLVM_SRC_URL https://github.com/llvm/llvm-project/archive/llvmorg-${VER}.tar.gz +ENV LLVM_TESTS_SRC_URL https://github.com/llvm/llvm-test-suite/archive/llvmorg-${VER}.tar.gz ENV MUSL_SRC_URL https://github.com/quic/musl/archive/d125203fcb134febcde6ca32181554560b67c790.tar.gz ENV HEXMVM_SRC_URL https://github.com/quic/hexagonMVM/archive/v0.1.1.tar.gz ENV LINUX_SRC_URL https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-6.7.11.tar.xz diff --git a/get-src-tarballs.sh b/get-src-tarballs.sh index d6bf6aa..4597a57 100755 --- a/get-src-tarballs.sh +++ b/get-src-tarballs.sh @@ -17,6 +17,14 @@ get_src_tarballs() { echo ${LLVM_SRC_URL} > ${MANIFEST_DIR}/llvm-project.txt cd - + wget --quiet ${LLVM_TESTS_SRC_URL} -O llvm-test-suite.tar.xz + mkdir llvm-test-suite + cd llvm-test-suite + tar xf ../llvm-test-suite.tar.xz --strip-components=1 + rm ../llvm-test-suite.tar.xz + echo ${LLVM_TESTS_SRC_URL} > ${MANIFEST_DIR}/llvm-test-suite.txt + cd - + git clone --branch ${QEMU_REF} ${QEMU_REPO} cd qemu git remote -v > ${MANIFEST_DIR}/qemu.txt From 004e5fcd74208575f286b96b29f4a9b0432c874e Mon Sep 17 00:00:00 2001 From: Brian Cain Date: Sat, 5 Oct 2024 22:19:59 -0500 Subject: [PATCH 10/10] Add psutil package When testing the toolchain with the llvm-test-suite, llvm-lit complains: Setting a timeout per test requires the Python psutil module but it could not be found. Try installing it via pip or via your operating system's package manager. Add the psutil package to workaround this failure. Signed-off-by: Brian Cain --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index fd814df..db364b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,6 +45,7 @@ RUN apt update && \ bc \ ninja-build \ cpio \ + python3-psutil \ unzip RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list