add --use-built-in-libdl option to component link subcommand (#1478)

* add `--use-built-in-libdl` option to `component link` subcommand This tells `wit-component` to include a prebuilt libdl.so when linking. This library provides `dlopen`, `dlsym`, etc. using the lookup tables generated by the linker when one or more `--dl-openable` options are provided. Signed-off-by: Joel Dice <[email protected]> * mark `dl` crate unpublishable Signed-off-by: Joel Dice <[email protected]> * move libdl.so to hopefully make `./publish verify` happy Signed-off-by: Joel Dice <[email protected]> * update link-dl-openable-builtin-libdl/component.wat Signed-off-by: Joel Dice <[email protected]> * tweak error message set by `dlsym` Signed-off-by: Joel Dice <[email protected]> * update libdl.so and link-dl-openable-builtin-libdl/component.wat Signed-off-by: Joel Dice <[email protected]> * address review feedback - make libdl.so smaller - remove unnecessary newtype - check that libdl.so is up-to-date during CI - use CStr literal syntax - add `dl` crate to workspace Signed-off-by: Joel Dice <[email protected]> * CI fix Signed-off-by: Joel Dice <[email protected]> * use LTO and strip when building libdl.so This helps ensure the build is deterministic and not system-dependent. Signed-off-by: Joel Dice <[email protected]> * another CI fix Signed-off-by: Joel Dice <[email protected]> * exclude `dl` crate when testing on WebAssembly Signed-off-by: Joel Dice <[email protected]> * remove debug logging Signed-off-by: Joel Dice <[email protected]> * revert unneeded change in crates/wit-component/tests/components.rs Signed-off-by: Joel Dice <[email protected]> --------- Signed-off-by: Joel Dice <[email protected]>
bytecodealliance · Mar 29, 2024 · 20874d5 · 20874d5
1 parent cbf7c01
commit 20874d5
Show file tree

Hide file tree

Showing 19 changed files with 962 additions and 5 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -79,6 +79,13 @@ jobs:
         submodules: true
     - name: Install Rust (rustup)
       run: rustup update ${{ matrix.rust }} --no-self-update && rustup default ${{ matrix.rust }}
+    - run: |
+        curl -LO https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/wasi-sdk-21.0-linux.tar.gz
+        tar xf wasi-sdk-21.0-linux.tar.gz
+        export WASI_SDK_PATH=$(pwd)/wasi-sdk-21.0
+        rustup target add wasm32-wasi
+        cd crates/wit-component/dl && bash check.sh
+      if: matrix.os == 'ubuntu-latest' && matrix.build == 'stable'
     - run: cargo test --locked --all
     - run: cargo test --locked -p wasmparser --benches
     - run: cargo test --locked -p wasm-encoder --all-features
@@ -109,7 +116,8 @@ jobs:
         cargo --locked test --workspace \
           --exclude fuzz-stats \
           --exclude wasm-tools-fuzz \
-          --exclude wasm-mutate-stats
+          --exclude wasm-mutate-stats \
+          --exclude dl
 
   rustfmt:
     name: Rustfmt

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -22,6 +22,7 @@ members = [
   'crates/wasm-mutate-stats',
   'fuzz',
   'crates/wit-parser/fuzz',
+  'crates/wit-component/dl',
 ]
 
 [workspace.lints.rust]

diff --git a/crates/wit-component/dl/Cargo.toml b/crates/wit-component/dl/Cargo.toml
@@ -0,0 +1,10 @@
+[package]
+name = "dl"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+crate-type = [ "staticlib" ]
+
+[dependencies]
diff --git a/crates/wit-component/dl/build.sh b/crates/wit-component/dl/build.sh
@@ -0,0 +1,10 @@
+# This script will rebuild libdl.so from source.
+#
+# To run this, you'll need to use `wasi-sdk` 21 or later, installed at
+# $WASI_SDK_PATH.
+#
+# Example: WASI_SDK_PATH=/opt/wasi-sdk bash build.sh ../libdl.so
+
+CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C relocation-model=pic" cargo build --release --target=wasm32-wasi
+$WASI_SDK_PATH/bin/clang -shared -o $1 -Wl,--whole-archive ../../../target/wasm32-wasi/release/libdl.a -Wl,--no-whole-archive
+cargo run --manifest-path ../../../Cargo.toml -- strip -a $1 -o $1
diff --git a/crates/wit-component/dl/check.sh b/crates/wit-component/dl/check.sh
@@ -0,0 +1,7 @@
+bash ./build.sh ../../../target/wasm32-wasi/release/tmp.so
+if diff ../../../target/wasm32-wasi/release/tmp.so ../libdl.so; then
+  exit 0
+else
+  echo "libdl.so is out-of-date; please run crates/wit-component/dl/build.sh to update it">&2
+  exit 1
+fi
diff --git a/crates/wit-component/dl/src/lib.rs b/crates/wit-component/dl/src/lib.rs
@@ -0,0 +1,181 @@
+#![no_std]
+
+//! This library uses lookup tables generated by `wasm-tools component link` to
+//! provide runtime library and symbol resolution via `dlopen` and `dlsym`.  The
+//! tables are provided via a call to `__wasm_set_libraries`, which `wasm-tools
+//! component link` arranges as part of component instantiation.
+
+use core::{
+    ffi::{c_char, c_int, c_void, CStr},
+    ptr, slice,
+};
+
+const RTLD_LAZY: c_int = 1;
+const RTLD_NOW: c_int = 2;
+
+const RTLD_NEXT: isize = -1;
+const RTLD_DEFAULT: isize = 0;
+
+#[repr(C)]
+pub struct Name {
+    length: u32,
+    data: *const u8,
+}
+
+#[repr(C)]
+pub struct Symbol {
+    name: Name,
+    address: *const c_void,
+}
+
+#[repr(C)]
+pub struct Symbols {
+    count: u32,
+    symbols: *const Symbol,
+}
+
+#[repr(C)]
+pub struct Library {
+    name: Name,
+    symbols: Symbols,
+}
+
+#[repr(C)]
+pub struct Libraries {
+    count: u32,
+    libraries: *const Library,
+}
+
+static mut ERROR: *const c_char = ptr::null();
+static mut LIBRARIES: *const Libraries = ptr::null();
+
+unsafe fn invalid_handle(library: *const c_void) -> bool {
+    if LIBRARIES.is_null() {
+        panic!(
+            "`__wasm_set_libraries` should have been called during \
+             instantiation with a non-NULL value"
+        );
+    }
+
+    let library = library as *const Library;
+    if (0..(*LIBRARIES).count)
+        .any(|index| (*LIBRARIES).libraries.add(usize::try_from(index).unwrap()) == library)
+    {
+        false
+    } else {
+        ERROR = c"invalid library handle".as_ptr();
+        true
+    }
+}
+
+/// # Safety
+///
+/// `library` must be a valid, not-yet-closed library pointer returned by
+/// `dlopen`.
+#[no_mangle]
+pub unsafe extern "C" fn dlclose(library: *mut c_void) -> c_int {
+    if invalid_handle(library) {
+        -1
+    } else {
+        0
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn dlerror() -> *const c_char {
+    unsafe {
+        let value = ERROR;
+        ERROR = ptr::null();
+        value
+    }
+}
+
+/// # Safety
+///
+/// `name` must be a valid pointer to a null-terminated string.
+#[no_mangle]
+pub unsafe extern "C" fn dlopen(name: *const c_char, flags: c_int) -> *const c_void {
+    if LIBRARIES.is_null() {
+        panic!(
+            "`__wasm_set_libraries` should have been called during \
+             instantiation with a non-NULL value"
+        );
+    }
+
+    if (flags & !(RTLD_LAZY | RTLD_NOW)) != 0 {
+        // TODO
+        ERROR = c"dlopen flags not yet supported".as_ptr();
+        return ptr::null();
+    }
+
+    let name = CStr::from_ptr(name);
+    let name = name.to_bytes();
+    let libraries = slice::from_raw_parts(
+        (*LIBRARIES).libraries,
+        usize::try_from((*LIBRARIES).count).unwrap(),
+    );
+    if let Ok(index) = libraries.binary_search_by(|library| {
+        slice::from_raw_parts(
+            library.name.data,
+            usize::try_from(library.name.length).unwrap(),
+        )
+        .cmp(name)
+    }) {
+        &libraries[index] as *const _ as _
+    } else {
+        ERROR = c"library not found".as_ptr();
+        ptr::null()
+    }
+}
+
+/// # Safety
+///
+/// `library` must be a valid, not-yet-closed library pointer returned by
+/// `dlopen`, and `name` must be a valid pointer to a null-terminated string.
+#[no_mangle]
+pub unsafe extern "C" fn dlsym(library: *const c_void, name: *const c_char) -> *const c_void {
+    if library as isize == RTLD_NEXT || library as isize == RTLD_DEFAULT {
+        // TODO
+        ERROR = c"dlsym RTLD_NEXT and RTLD_DEFAULT not yet supported".as_ptr();
+        return ptr::null();
+    }
+
+    if invalid_handle(library) {
+        return ptr::null();
+    }
+
+    let library = library as *const Library;
+    let name = CStr::from_ptr(name);
+    let name = name.to_bytes();
+    let symbols = slice::from_raw_parts(
+        (*library).symbols.symbols,
+        usize::try_from((*library).symbols.count).unwrap(),
+    );
+    if let Ok(index) = symbols.binary_search_by(|symbol| {
+        slice::from_raw_parts(
+            symbol.name.data,
+            usize::try_from(symbol.name.length).unwrap(),
+        )
+        .cmp(name)
+    }) {
+        symbols[index].address
+    } else {
+        ERROR = c"symbol not found".as_ptr();
+        ptr::null()
+    }
+}
+
+/// # Safety
+///
+/// `libraries` must be a valid pointer to a `Libraries` object, and this
+/// pointer must remain valid for the lifetime of the process.
+#[no_mangle]
+pub unsafe extern "C" fn __wasm_set_libraries(libraries: *const Libraries) {
+    LIBRARIES = libraries;
+}
+
+#[cfg(target_arch = "wasm32")]
+#[panic_handler]
+fn panic(_info: &core::panic::PanicInfo) -> ! {
+    core::arch::wasm32::unreachable()
+}
diff --git a/crates/wit-component/libdl.so b/crates/wit-component/libdl.so
diff --git a/crates/wit-component/src/linking.rs b/crates/wit-component/src/linking.rs
@@ -57,9 +57,7 @@ enum Address<'a> {
 /// Represents a `dlopen`/`dlsym` lookup table enabling runtime symbol resolution
 ///
 /// The top level of this table is a sorted list of library names and offsets, each pointing to a sorted list of
-/// symbol names and offsets.  See
-/// https://github.com/dicej/wasi-libc/blob/76c7e1e1cfdad577ecd7f61c67ead7a38d62a7c4/libc-top-half/musl/src/misc/dl.c
-/// for how this is used.
+/// symbol names and offsets.  See ../dl/src/lib.rs for how this is used at runtime.
 struct DlOpenables<'a> {
     /// Offset into the main module's table where function references will be stored
     table_base: u32,
@@ -1202,6 +1200,9 @@ pub struct Linker {
     /// Whether to generate trapping stubs for any unresolved imports
     stub_missing_functions: bool,
 
+    /// Whether to use a built-in implementation of `dlopen`/`dlsym`.
+    use_built_in_libdl: bool,
+
     /// Size of stack (in bytes) to allocate in the synthesized main module
     ///
     /// If `None`, use `DEFAULT_STACK_SIZE_BYTES`.
@@ -1247,8 +1248,18 @@ impl Linker {
         self
     }
 
+    /// Specify whether to use a built-in implementation of `dlopen`/`dlsym`.
+    pub fn use_built_in_libdl(mut self, use_built_in_libdl: bool) -> Self {
+        self.use_built_in_libdl = use_built_in_libdl;
+        self
+    }
+
     /// Encode the component and return the bytes
     pub fn encode(mut self) -> Result<Vec<u8>> {
+        if self.use_built_in_libdl {
+            self = self.library("libdl.so", include_bytes!("../libdl.so"), false)?;
+        }
+
         let adapter_names = self
             .adapters
             .iter()
@@ -1332,6 +1343,7 @@ impl Linker {
                         .all(|(_, export)| export.flags.contains(SymbolFlags::BINDING_WEAK)))
             {
                 self.stub_missing_functions = false;
+                self.use_built_in_libdl = false;
                 self.libraries.push((
                     "wit-component:stubs".into(),
                     make_stubs_module(&missing),

diff --git a/crates/wit-component/tests/components.rs b/crates/wit-component/tests/components.rs
@@ -31,6 +31,9 @@ use wit_parser::{PackageId, Resolve, UnresolvedPackage};
 /// * [optional] `stub-missing-functions` - if linking libraries and this file
 ///   exists, `Linker::stub_missing_functions` will be set to `true`.  The
 ///   contents of the file are ignored.
+/// * [optional] `use-built-in-libdl` - if linking libraries and this file
+///   exists, `Linker::use_built_in_libdl` will be set to `true`.  The contents
+///   of the file are ignored.
 ///
 /// And the output files are one of the following:
 ///
@@ -94,6 +97,10 @@ fn component_encoding_via_flags() -> Result<()> {
                 linker = linker.stub_missing_functions(true);
             }
 
+            if path.join("use-built-in-libdl").is_file() {
+                linker = linker.use_built_in_libdl(true);
+            }
+
             let linker =
                 libs.into_iter()
                     .try_fold(linker, |linker, (prefix, path, dl_openable)| {