From a35b453a87314f126bdd3f554b084d273e3265fb Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Mon, 11 Dec 2023 17:20:52 -0700 Subject: [PATCH] add shared library support (#338) This adds support for building WASI shared libraries per https://github.com/WebAssembly/tool-conventions/blob/main/DynamicLinking.md. For the time being, the goal is to allow "pseudo-dynamic" linking using the Component Model per https://github.com/WebAssembly/component-model/blob/main/design/mvp/examples/SharedEverythingDynamicLinking.md. This requires all libraries to be available when the component is created, but still allows runtime symbol resolution via `dlopen`/`dlsym` backed by a static lookup table. This is sufficient to support Python native extensions, for example. A complete demo using `wit-component` is available at https://github.com/dicej/component-linking-demo. Signed-off-by: Joel Dice --- .gitmodules | 2 +- Makefile | 31 ++++++++++++++++++------------- src/wasi-libc | 2 +- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.gitmodules b/.gitmodules index e8b2eace4..0691635ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,7 +3,7 @@ url = https://github.com/llvm/llvm-project [submodule "src/wasi-libc"] path = src/wasi-libc - url = https://github.com/CraneStation/wasi-libc + url = https://github.com/WebAssembly/wasi-libc [submodule "src/config"] path = src/config url = https://git.savannah.gnu.org/git/config.git diff --git a/Makefile b/Makefile index 319cabd90..fd705b3a1 100644 --- a/Makefile +++ b/Makefile @@ -107,12 +107,13 @@ build/llvm.BUILT: llvm-config touch build/llvm.BUILT -build/wasi-libc.BUILT: build/llvm.BUILT +build/wasi-libc.BUILT: build/compiler-rt.BUILT $(MAKE) -C $(ROOT_DIR)/src/wasi-libc \ CC=$(BUILD_PREFIX)/bin/clang \ AR=$(BUILD_PREFIX)/bin/llvm-ar \ NM=$(BUILD_PREFIX)/bin/llvm-nm \ - SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot + SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ + default libc_so $(MAKE) -C $(ROOT_DIR)/src/wasi-libc \ CC=$(BUILD_PREFIX)/bin/clang \ AR=$(BUILD_PREFIX)/bin/llvm-ar \ @@ -121,7 +122,7 @@ build/wasi-libc.BUILT: build/llvm.BUILT THREAD_MODEL=posix touch build/wasi-libc.BUILT -build/compiler-rt.BUILT: build/llvm.BUILT build/wasi-libc.BUILT +build/compiler-rt.BUILT: build/llvm.BUILT # Do the build, and install it. mkdir -p build/compiler-rt cd build/compiler-rt && cmake -G Ninja \ @@ -151,6 +152,8 @@ build/compiler-rt.BUILT: build/llvm.BUILT build/wasi-libc.BUILT touch build/compiler-rt.BUILT # Flags for libcxx and libcxxabi. +# $(1): pthreads ON or OFF +# $(2): shared libraries ON or OFF LIBCXX_CMAKE_FLAGS = \ -DCMAKE_C_COMPILER_WORKS=ON \ -DCMAKE_CXX_COMPILER_WORKS=ON \ @@ -158,41 +161,43 @@ LIBCXX_CMAKE_FLAGS = \ -DCMAKE_MODULE_PATH=$(ROOT_DIR)/cmake \ -DCMAKE_TOOLCHAIN_FILE=$(ROOT_DIR)/wasi-sdk.cmake \ -DCMAKE_STAGING_PREFIX=$(PREFIX)/share/wasi-sysroot \ + -DCMAKE_POSITION_INDEPENDENT_CODE=$(2) \ -DLLVM_CONFIG_PATH=$(ROOT_DIR)/build/llvm/bin/llvm-config \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \ -DCXX_SUPPORTS_CXX11=ON \ - -DLIBCXX_ENABLE_THREADS:BOOL=@PTHREAD@ \ - -DLIBCXX_HAS_PTHREAD_API:BOOL=@PTHREAD@ \ + -DLIBCXX_ENABLE_THREADS:BOOL=$(1) \ + -DLIBCXX_HAS_PTHREAD_API:BOOL=$(1) \ -DLIBCXX_HAS_EXTERNAL_THREAD_API:BOOL=OFF \ -DLIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL=OFF \ -DLIBCXX_HAS_WIN32_THREAD_API:BOOL=OFF \ -DLLVM_COMPILER_CHECKED=ON \ -DCMAKE_BUILD_TYPE=RelWithDebugInfo \ - -DLIBCXX_ENABLE_SHARED:BOOL=OFF \ + -DLIBCXX_ENABLE_SHARED:BOOL=$(2) \ -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY:BOOL=OFF \ -DLIBCXX_ENABLE_EXCEPTIONS:BOOL=OFF \ -DLIBCXX_ENABLE_FILESYSTEM:BOOL=OFF \ + -DLIBCXX_ENABLE_ABI_LINKER_SCRIPT:BOOL=OFF \ -DLIBCXX_CXX_ABI=libcxxabi \ -DLIBCXX_CXX_ABI_INCLUDE_PATHS=$(LLVM_PROJ_DIR)/libcxxabi/include \ -DLIBCXX_HAS_MUSL_LIBC:BOOL=ON \ -DLIBCXX_ABI_VERSION=2 \ -DLIBCXXABI_ENABLE_EXCEPTIONS:BOOL=OFF \ - -DLIBCXXABI_ENABLE_SHARED:BOOL=OFF \ + -DLIBCXXABI_ENABLE_SHARED:BOOL=$(2) \ -DLIBCXXABI_SILENT_TERMINATE:BOOL=ON \ - -DLIBCXXABI_ENABLE_THREADS:BOOL=@PTHREAD@ \ - -DLIBCXXABI_HAS_PTHREAD_API:BOOL=@PTHREAD@ \ + -DLIBCXXABI_ENABLE_THREADS:BOOL=$(1) \ + -DLIBCXXABI_HAS_PTHREAD_API:BOOL=$(1) \ -DLIBCXXABI_HAS_EXTERNAL_THREAD_API:BOOL=OFF \ -DLIBCXXABI_BUILD_EXTERNAL_THREAD_LIBRARY:BOOL=OFF \ -DLIBCXXABI_HAS_WIN32_THREAD_API:BOOL=OFF \ - -DLIBCXXABI_ENABLE_PIC:BOOL=OFF \ + -DLIBCXXABI_ENABLE_PIC:BOOL=$(2) \ -DWASI_SDK_PREFIX=$(BUILD_PREFIX) \ -DUNIX:BOOL=ON \ --debug-trycompile -build/libcxx.BUILT: build/llvm.BUILT build/compiler-rt.BUILT build/wasi-libc.BUILT +build/libcxx.BUILT: build/llvm.BUILT build/wasi-libc.BUILT # Do the build. mkdir -p build/libcxx - cd build/libcxx && cmake -G Ninja $(LIBCXX_CMAKE_FLAGS:@PTHREAD@=OFF) \ + cd build/libcxx && cmake -G Ninja $(call LIBCXX_CMAKE_FLAGS,OFF,ON) \ -DCMAKE_SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ -DCMAKE_C_FLAGS="$(DEBUG_PREFIX_MAP) $(EXTRA_CFLAGS)" \ -DCMAKE_CXX_FLAGS="$(DEBUG_PREFIX_MAP) $(EXTRA_CXXFLAGS)" \ @@ -202,7 +207,7 @@ build/libcxx.BUILT: build/llvm.BUILT build/compiler-rt.BUILT build/wasi-libc.BUI $(LLVM_PROJ_DIR)/runtimes ninja $(NINJA_FLAGS) -C build/libcxx mkdir -p build/libcxx-threads - cd build/libcxx-threads && cmake -G Ninja $(LIBCXX_CMAKE_FLAGS:@PTHREAD@=ON) \ + cd build/libcxx-threads && cmake -G Ninja $(call LIBCXX_CMAKE_FLAGS,ON,OFF) \ -DCMAKE_SYSROOT=$(BUILD_PREFIX)/share/wasi-sysroot \ -DCMAKE_C_FLAGS="$(DEBUG_PREFIX_MAP) -pthread $(EXTRA_CFLAGS)" \ -DCMAKE_CXX_FLAGS="$(DEBUG_PREFIX_MAP) -pthread $(EXTRA_CXXFLAGS)" \ diff --git a/src/wasi-libc b/src/wasi-libc index ec4566bea..9c17f5235 160000 --- a/src/wasi-libc +++ b/src/wasi-libc @@ -1 +1 @@ -Subproject commit ec4566beae84e54952637f0bf61bee4b4cacc087 +Subproject commit 9c17f5235c7977cb2a000990eb8c605a25a80adf