Skip to content

Commit

Permalink
add WASI Preview 2 bindings (#460)
Browse files Browse the repository at this point in the history 
* add WASI Preview 2 bindings

This adds C bindings generated from the `wasi:cli/[email protected]`
world, plus a makefile target to regenerate them from the WIT source files.

We'll use these bindings to call Preview 2 host functions when building for the
`wasm32-wasi-preview2` target.

Signed-off-by: Joel Dice <[email protected]>

* update to pre-release of `wit-bindgen` 0.17.0

This includes bytecodealliance/wit-bindgen#804 (fix
broken indentation in generated code) and
bytecodealliance/wit-bindgen#805 (support overriding
world name and adding a suffix to the component type custom section).

Signed-off-by: Joel Dice <[email protected]>

* test all targets; update preview2 expected output files

Signed-off-by: Joel Dice <[email protected]>

* build for `wasm32-wasi-threads` before testing it

Signed-off-by: Joel Dice <[email protected]>

* move generated bindings per review feedback

Since these files aren't part of cloudlibc, no reason to put them under the
cloudlibc directory.

Signed-off-by: Joel Dice <[email protected]>

* move preview2.h to wasi directory

Signed-off-by: Joel Dice <[email protected]>

---------

Signed-off-by: Joel Dice <[email protected]>
  • Loading branch information
@dicej
dicej authored Jan 11, 2024
1 parent 03b228e commit 47b9db6
Show file tree
Hide file tree
Showing 10 changed files with 7,502 additions and 25 deletions.
22 changes: 12 additions & 10 deletions .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,14 @@ jobs:
make -j4
WASI_SNAPSHOT=preview2 make -j4
- name: Build libc + threads
# Only build the thread-capable wasi-libc in the latest supported Clang
# version; the earliest version does not have all necessary builtins
# (e.g., `__builtin_wasm_memory_atomic_notify`).
if: matrix.clang_version != '10.0.0'
shell: bash
run: make -j4 THREAD_MODEL=posix

- name: Test
shell: bash
# For Clang linking to work correctly, we need to place Clang's runtime
Expand All @@ -99,23 +107,17 @@ jobs:
export WASI_DIR=$(realpath $($CLANG_DIR/clang -print-resource-dir)/lib/wasi/)
mkdir -p $WASI_DIR
cp download/lib/wasi/libclang_rt.builtins-wasm32.a $WASI_DIR
make test
TARGET_TRIPLE=wasm32-wasi make test
rm -r build
TARGET_TRIPLE=wasm32-wasi-preview2 make test
rm -r build
WASI_SNAPSHOT=preview2 make test
TARGET_TRIPLE=wasm32-wasi-threads make test
# The older version of Clang does not provide the expected symbol for the
# test entrypoints: `undefined symbol: __main_argc_argv`.
# The older (<15.0.7) version of wasm-ld does not provide `__heap_end`,
# which is required by our malloc implementation.
if: matrix.os == 'ubuntu-latest' && matrix.clang_version != '10.0.0'

- name: Build libc + threads
# Only build the thread-capable wasi-libc in the latest supported Clang
# version; the earliest version does not have all necessary builtins
# (e.g., `__builtin_wasm_memory_atomic_notify`).
if: matrix.clang_version != '10.0.0'
shell: bash
run: make -j4 THREAD_MODEL=posix

- uses: actions/upload-artifact@v1
with:
# Upload the sysroot folder. Give it a name according to the OS it was built for.
Expand Down
84 changes: 81 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,17 @@ MALLOC_IMPL ?= dlmalloc
BUILD_LIBC_TOP_HALF ?= yes
# The directory where we will store intermediate artifacts.
OBJDIR ?= build/$(TARGET_TRIPLE)
# The directory where we store files and tools for generating WASI Preview 2 bindings
BINDING_WORK_DIR ?= build/bindings
# URL from which to retrieve the WIT files used to generate the WASI Preview 2 bindings
WASI_CLI_URL ?= https://github.com/WebAssembly/wasi-cli/archive/refs/tags/v0.2.0-rc-2023-12-05.tar.gz
# URL from which to retrieve the `wit-bindgen` command used to generate the WASI
# Preview 2 bindings.
#
# TODO: Switch to bytecodealliance/wit-bindgen 0.17.0 once it's released (which
# will include https://github.com/bytecodealliance/wit-bindgen/pull/804 and
# https://github.com/bytecodealliance/wit-bindgen/pull/805, which we rely on)
WIT_BINDGEN_URL ?= https://github.com/dicej/wit-bindgen/releases/download/wit-bindgen-cli-0.17.0-dicej-pre0/wit-bindgen-v0.17.0-dicej-pre0-x86_64-linux.tar.gz

# When the length is no larger than this threshold, we consider the
# overhead of bulk memory opcodes to outweigh the performance benefit,
Expand All @@ -37,7 +48,7 @@ BULK_MEMORY_THRESHOLD ?= 32
# Set the default WASI target triple.
TARGET_TRIPLE = wasm32-wasi

# Threaded version necessitates a different traget, as objects from different
# Threaded version necessitates a different target, as objects from different
# targets can't be mixed together while linking.
ifeq ($(THREAD_MODEL), posix)
TARGET_TRIPLE = wasm32-wasi-threads
Expand Down Expand Up @@ -68,6 +79,16 @@ LIBC_BOTTOM_HALF_ALL_SOURCES = \
$(shell find $(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC) -name \*.c) \
$(shell find $(LIBC_BOTTOM_HALF_SOURCES) -name \*.c))

ifeq ($(WASI_SNAPSHOT), preview1)
# Omit source files not relevant to WASI Preview 1. As we introduce files
# supporting `wasi-sockets` for `wasm32-wasi-preview2`, we'll add those files to
# this list.
LIBC_BOTTOM_HALF_OMIT_SOURCES := $(LIBC_BOTTOM_HALF_SOURCES)/preview2.c
LIBC_BOTTOM_HALF_ALL_SOURCES := $(filter-out $(LIBC_BOTTOM_HALF_OMIT_SOURCES),$(LIBC_BOTTOM_HALF_ALL_SOURCES))
# Omit preview2.h from include-all.c test.
INCLUDE_ALL_CLAUSES := -not -name preview2.h
endif

# FIXME(https://reviews.llvm.org/D85567) - due to a bug in LLD the weak
# references to a function defined in `chdir.c` only work if `chdir.c` is at the
# end of the archive, but once that LLD review lands and propagates into LLVM
Expand Down Expand Up @@ -741,7 +762,7 @@ check-symbols: startup_files libc
# Generate a test file that includes all public C header files.
#
cd "$(SYSROOT_INC)" && \
for header in $$(find . -type f -not -name mman.h -not -name signal.h -not -name times.h -not -name resource.h |grep -v /bits/ |grep -v /c++/); do \
for header in $$(find . -type f -not -name mman.h -not -name signal.h -not -name times.h -not -name resource.h $(INCLUDE_ALL_CLAUSES) |grep -v /bits/ |grep -v /c++/); do \
echo '#include <'$$header'>' | sed 's/\.\///' ; \
done |LC_ALL=C sort >$(SYSROOT_SHARE)/include-all.c ; \
cd - >/dev/null
Expand Down Expand Up @@ -816,8 +837,65 @@ install: finish
mkdir -p "$(INSTALL_DIR)"
cp -r "$(SYSROOT)/lib" "$(SYSROOT)/share" "$(SYSROOT)/include" "$(INSTALL_DIR)"

$(BINDING_WORK_DIR)/wasi-cli:
mkdir -p "$(BINDING_WORK_DIR)"
cd "$(BINDING_WORK_DIR)" && \
curl -L "$(WASI_CLI_URL)" -o wasi-cli.tar.gz && \
tar xf wasi-cli.tar.gz && \
mv wasi-cli-* wasi-cli

$(BINDING_WORK_DIR)/wit-bindgen:
mkdir -p "$(BINDING_WORK_DIR)"
cd "$(BINDING_WORK_DIR)" && \
curl -L "$(WIT_BINDGEN_URL)" -o wit-bindgen.tar.gz && \
tar xf wit-bindgen.tar.gz && \
mv wit-bindgen-* wit-bindgen

bindings: $(BINDING_WORK_DIR)/wasi-cli $(BINDING_WORK_DIR)/wit-bindgen
cd "$(BINDING_WORK_DIR)" && \
./wit-bindgen/wit-bindgen c \
--rename-world preview2 \
--type-section-suffix __wasi_libc \
--world wasi:cli/[email protected] \
--rename wasi:clocks/[email protected]=monotonic_clock \
--rename wasi:clocks/[email protected]=wall_clock \
--rename wasi:filesystem/[email protected]=filesystem_preopens \
--rename wasi:filesystem/[email protected]=filesystem \
--rename wasi:io/[email protected]=io_error \
--rename wasi:io/[email protected]=poll \
--rename wasi:io/[email protected]=streams \
--rename wasi:random/[email protected]=random_insecure_seed \
--rename wasi:random/[email protected]=random_insecure \
--rename wasi:random/[email protected]=random \
--rename wasi:sockets/[email protected]=instance_network \
--rename wasi:sockets/[email protected]=ip_name_lookup \
--rename wasi:sockets/[email protected]=network \
--rename wasi:sockets/[email protected]=tcp_create_socket \
--rename wasi:sockets/[email protected]=tcp \
--rename wasi:sockets/[email protected]=udp_create_socket \
--rename wasi:sockets/[email protected]=udp \
--rename wasi:cli/[email protected]=environment \
--rename wasi:cli/[email protected]=exit \
--rename wasi:cli/[email protected]=stdin \
--rename wasi:cli/[email protected]=stdout \
--rename wasi:cli/[email protected]=stderr \
--rename wasi:cli/[email protected]=terminal_input \
--rename wasi:cli/[email protected]=terminal_output \
--rename wasi:cli/[email protected]=terminal_stdin \
--rename wasi:cli/[email protected]=terminal_stdout \
--rename wasi:cli/[email protected]=terminal_stderr \
./wasi-cli/wit && \
mv preview2.h ../../libc-bottom-half/headers/public/wasi/ && \
mv preview2_component_type.o ../../libc-bottom-half/sources && \
sed 's_#include "preview2.h"_#include "wasi/preview2.h"_' \
< preview2.c \
> ../../libc-bottom-half/sources/preview2.c && \
rm preview2.c


clean:
$(RM) -r "$(BINDING_WORK_DIR)"
$(RM) -r "$(OBJDIR)"
$(RM) -r "$(SYSROOT)"

.PHONY: default startup_files libc libc_so finish install include_dirs clean check-symbols
.PHONY: default startup_files libc libc_so finish install include_dirs clean check-symbols bindings
Loading

0 comments on commit 47b9db6

Please sign in to comment.