From 1c134bb6e8cfa54ebf6a277b1e46cda965cbc2c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Zemanovi=C4=8D?= Date: Mon, 22 Jul 2024 14:15:45 +0100 Subject: [PATCH] workaround wasmer store segfault by keeping it in cache --- Cargo.lock | 52 ++++--- Cargo.toml | 10 +- crates/namada/Cargo.toml | 2 + crates/namada/src/vm/host_env.rs | 12 +- .../src/vm/wasm/compilation_cache/common.rs | 137 ++++++++++-------- crates/namada/src/vm/wasm/run.rs | 130 ++++++++++++++--- crates/tests/Cargo.toml | 3 +- crates/tests/src/vm_host_env/tx.rs | 17 ++- wasm/Cargo.lock | 50 ++++--- wasm_for_tests/Cargo.lock | 131 ++++------------- 10 files changed, 304 insertions(+), 240 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1018527284..6cb29ae6d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4426,10 +4426,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] @@ -4729,7 +4729,9 @@ dependencies = [ "wasm-instrument", "wasmer", "wasmer-cache", + "wasmer-compiler", "wasmer-compiler-singlepass", + "wasmer-types", "wasmer-vm", "wasmparser 0.107.0", "wasmtimer", @@ -5434,6 +5436,7 @@ dependencies = [ "test-log", "tokio", "toml 0.5.11", + "wasmer", ] [[package]] @@ -6803,14 +6806,14 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "region" -version = "3.0.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e189c2369884dce920945e2ddf79b3dff49e071a167dd1817fa9c4c00d512e" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" dependencies = [ "bitflags 1.3.2", "libc", - "mach", - "winapi", + "mach2", + "windows-sys 0.52.0", ] [[package]] @@ -9123,9 +9126,9 @@ dependencies = [ [[package]] name = "wasmer" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1852ee143a2d8143265bfee017c43bf690702d6c2b45a763a2f13e669f5b7ec" +checksum = "c3a6e0f73e5ae361fe64db607eaf4ab2381d88ad2c1b0bb8cf254cf35d894687" dependencies = [ "bytes", "cfg-if", @@ -9152,9 +9155,9 @@ dependencies = [ [[package]] name = "wasmer-cache" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "092ca4da29a7320b99c9b6660dfbde055f4db0a71c1273860272c213b2f7e019" +checksum = "79fd0889f8844b7c70b8ee8fbf1d1f6ccff99399c6f3d3627048cde04b1ac493" dependencies = [ "blake3", "hex", @@ -9164,9 +9167,9 @@ dependencies = [ [[package]] name = "wasmer-compiler" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4f157d715f3bb60c2c9d7b9e48299a30e9209f87f4484f79f9cd586b40b6ee" +checksum = "cb1e7c79507f5d55f1afd99984717e8380440cd98e13d542e4d00661f986f2d4" dependencies = [ "backtrace", "bytes", @@ -9175,6 +9178,7 @@ dependencies = [ "enumset", "lazy_static", "leb128", + "libc", "memmap2 0.5.10", "more-asserts", "region", @@ -9192,9 +9196,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-cranelift" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb457e66b77ca2188fbbd6c2056ec6e8ccb4bddee73e60ba9d39733d7b2e8068" +checksum = "8f3352014573750327646a690d32774312b0e8b7920e7e8ba00c0449eac18390" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -9211,9 +9215,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-singlepass" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a3196b2a87d5c6692021ece7ad1cf7fe43b7f1669c3aba1b8ccfcebe660070c" +checksum = "cc490f011e855eb5e31475ff3e9efa83840cc0ed2f8322dfaca18627df0a9f3c" dependencies = [ "byteorder", "dynasm", @@ -9252,9 +9256,9 @@ dependencies = [ [[package]] name = "wasmer-derive" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cd5732ff64370e98986f9753cce13b91cc9d3c4b649e31b0d08d5db69164ea" +checksum = "ac6b0b0580cfa1fc7ad58cca3626a742f2b2e5ccd51cfc5de43e8edb0d1daa4c" dependencies = [ "proc-macro-error", "proc-macro2", @@ -9264,9 +9268,9 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c890fd0dbda40df03977b899d1ad7113deba3c225f2cc7b88deb7633044d3e07" +checksum = "576442cc3d302ca215fd40aa7826a078571dca7eaa773d8cdedca14a2ec7c9a1" dependencies = [ "bytecheck", "enum-iterator", @@ -9285,9 +9289,9 @@ dependencies = [ [[package]] name = "wasmer-vm" -version = "4.3.2" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e0dc60ab800cf0bd44e2d35d88422d256d2470b00c72778f91bfb826c42dbd0" +checksum = "6483035d1df84a978cd6c6a35878e913dc8ec6311f8712548a922a75e87957ba" dependencies = [ "backtrace", "cc", @@ -9301,7 +9305,7 @@ dependencies = [ "indexmap 1.9.3", "lazy_static", "libc", - "mach", + "mach2", "memoffset 0.9.0", "more-asserts", "region", diff --git a/Cargo.toml b/Cargo.toml index 6bb9135495..57e84392ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -191,10 +191,12 @@ tracing-log = "0.2.0" tracing-subscriber = {version = "0.3.7", default-features = false, features = ["env-filter", "fmt"]} wasmparser = "0.107.0" wasm-instrument = {version = "0.4.0", features = ["sign_ext"]} -wasmer = "4.3.1" -wasmer-cache = "4.3.1" -wasmer-compiler-singlepass = "4.3.1" -wasmer-vm = "4.3.1" +wasmer = "4.3.5" +wasmer-cache = "4.3.5" +wasmer-compiler = "4.3.5" +wasmer-compiler-singlepass = "4.3.5" +wasmer-types = "4.3.5" +wasmer-vm = "4.3.5" winapi = "0.3.9" yansi = "0.5.1" zeroize = { version = "1.5.5", features = ["zeroize_derive"] } diff --git a/crates/namada/Cargo.toml b/crates/namada/Cargo.toml index c728bade80..65ec56741c 100644 --- a/crates/namada/Cargo.toml +++ b/crates/namada/Cargo.toml @@ -169,3 +169,5 @@ tempfile.workspace = true test-log.workspace = true tokio = { workspace = true, features = ["rt", "macros"] } tracing-subscriber.workspace = true +wasmer-compiler = { workspace = true } +wasmer-types = { workspace = true } diff --git a/crates/namada/src/vm/host_env.rs b/crates/namada/src/vm/host_env.rs index 2db29825b9..38dab14943 100644 --- a/crates/namada/src/vm/host_env.rs +++ b/crates/namada/src/vm/host_env.rs @@ -2422,6 +2422,7 @@ pub mod testing { tx_index: &TxIndex, result_buffer: &mut Option>, yielded_value: &mut Option>, + store: Rc>, #[cfg(feature = "wasm-runtime")] vp_wasm_cache: &mut VpCache, #[cfg(feature = "wasm-runtime")] tx_wasm_cache: &mut TxCache, ) -> TxVmEnv::D, ::H, CA> @@ -2429,14 +2430,15 @@ pub mod testing { S: State, CA: WasmCacheAccess, { - let mut store = crate::vm::wasm::compilation_cache::common::store(); - - let wasm_memory = - crate::vm::wasm::memory::prepare_tx_memory(&mut store).unwrap(); + let wasm_memory = { + let mut borrowed_store = store.borrow_mut(); + crate::vm::wasm::memory::prepare_tx_memory(&mut *borrowed_store) + .unwrap() + }; let (write_log, in_mem, db) = state.split_borrow(); let mut env = TxVmEnv::new( - WasmMemory::new(Rc::new(RefCell::new(store))), + WasmMemory::new(Rc::downgrade(&store)), write_log, in_mem, db, diff --git a/crates/namada/src/vm/wasm/compilation_cache/common.rs b/crates/namada/src/vm/wasm/compilation_cache/common.rs index 146d03df3a..1f43d9ebdd 100644 --- a/crates/namada/src/vm/wasm/compilation_cache/common.rs +++ b/crates/namada/src/vm/wasm/compilation_cache/common.rs @@ -36,6 +36,12 @@ pub struct Cache { name: PhantomData, /// Cache access level access: PhantomData, + /// Wasmer store - The store that's used to compile the modules has to be + /// kept alive to avoid segfaults when running them from cache (otherwise + /// `wasmer_compiler::FRAME_INFO` gets cleared out when the `Store` is + /// dropped and the next run of this `Module` doesn't re-instantiate it and + /// crashes when it tries to access it) + store: Arc, } /// This trait is used to give names to different caches @@ -100,6 +106,7 @@ impl Cache { in_memory, name: Default::default(), access: Default::default(), + store: Arc::new(store()), } } @@ -111,11 +118,12 @@ impl Cache { &mut self, code_hash: &Hash, ) -> Result, wasm::run::Error> { - if A::is_read_write() { + let module = if A::is_read_write() { self.get(code_hash) } else { self.peek(code_hash) - } + }?; + Ok(module.map(|module| (module, store()))) } /// Get the current number of items in the cache @@ -130,10 +138,7 @@ impl Cache { /// Get a WASM module from LRU cache, from a file or compile it and cache /// it. Updates the position in the LRU cache. - fn get( - &mut self, - hash: &Hash, - ) -> Result, wasm::run::Error> { + fn get(&mut self, hash: &Hash) -> Result, wasm::run::Error> { let mut in_memory = self.in_memory.write().unwrap(); if let Some(module) = in_memory.get(hash) { tracing::trace!( @@ -141,7 +146,7 @@ impl Cache { N::name(), hash.to_string() ); - return Ok(Some((module.clone(), store()))); + return Ok(Some(module.clone())); } drop(in_memory); @@ -164,11 +169,11 @@ impl Cache { N::name(), hash.to_string() ); - return Ok(Some((module.clone(), store()))); + return Ok(Some(module.clone())); } - if let Ok((module, store)) = - file_load_module(&self.dir, hash) + if let Ok(module) = + file_load_module(&self.dir, hash, &self.store) { tracing::info!( "{} found {} in file cache.", @@ -179,7 +184,7 @@ impl Cache { let _ = in_memory.put_with_weight(*hash, module.clone()); - return Ok(Some((module, store))); + return Ok(Some(module)); } else { return Ok(None); } @@ -201,15 +206,16 @@ impl Cache { } None => { drop(progress); - let (module, store) = if module_file_exists(&self.dir, hash) - { + let module = if module_file_exists(&self.dir, hash) { tracing::info!( "Trying to load {} {} from file.", N::name(), hash.to_string() ); - if let Ok(res) = file_load_module(&self.dir, hash) { - res + if let Ok(module) = + file_load_module(&self.dir, hash, &self.store) + { + module } else { return Ok(None); } @@ -226,7 +232,7 @@ impl Cache { let mut in_memory = self.in_memory.write().unwrap(); let _ = in_memory.put_with_weight(*hash, module.clone()); - return Ok(Some((module, store))); + return Ok(Some(module)); } } } @@ -234,10 +240,7 @@ impl Cache { /// Peak-only is used for dry-ran txs (and VPs that the tx triggers). /// It doesn't update the in-memory cache. - fn peek( - &self, - hash: &Hash, - ) -> Result, wasm::run::Error> { + fn peek(&self, hash: &Hash) -> Result, wasm::run::Error> { let in_memory = self.in_memory.read().unwrap(); if let Some(module) = in_memory.peek(hash) { tracing::info!( @@ -245,7 +248,7 @@ impl Cache { N::name(), hash.to_string() ); - return Ok(Some((module.clone(), store()))); + return Ok(Some(module.clone())); } drop(in_memory); @@ -268,18 +271,18 @@ impl Cache { N::name(), hash.to_string() ); - return Ok(Some((module.clone(), store()))); + return Ok(Some(module.clone())); } - if let Ok((module, store)) = - file_load_module(&self.dir, hash) + if let Ok(module) = + file_load_module(&self.dir, hash, &self.store) { tracing::info!( "{} found {} in file cache.", N::name(), hash.to_string() ); - return Ok(Some((module, store))); + return Ok(Some(module)); } else { return Ok(None); } @@ -308,8 +311,10 @@ impl Cache { N::name(), hash.to_string() ); - if let Ok(res) = file_load_module(&self.dir, hash) { - return Ok(Some(res)); + if let Ok(module) = + file_load_module(&self.dir, hash, &self.store) + { + return Ok(Some(module)); } else { return Ok(None); } @@ -332,10 +337,14 @@ impl Cache { // It doesn't update the cache and files let progress = self.progress.read().unwrap(); match progress.get(&hash) { - Some(_) => return self.peek(&hash), + Some(_) => { + let module = self.peek(&hash)?; + return Ok(module.map(|module| (module, store()))); + } None => { let code = wasm::run::prepare_wasm_code(code)?; - return Ok(Some(compile(code)?)); + let module = compile(code, &self.store)?; + return Ok(Some((module, store()))); } } } @@ -351,8 +360,8 @@ impl Cache { tracing::info!("Compiling {} {}.", N::name(), hash.to_string()); match wasm::run::prepare_wasm_code(code) { - Ok(code) => match compile(code) { - Ok((module, store)) => { + Ok(code) => match compile(code, &self.store) { + Ok(module) => { // Write the file file_write_module(&self.dir, &module, &hash); @@ -364,7 +373,7 @@ impl Cache { let mut in_memory = self.in_memory.write().unwrap(); let _ = in_memory.put_with_weight(hash, module.clone()); - Ok(Some((module, store))) + Ok(Some((module, store()))) } Err(err) => { tracing::info!( @@ -410,13 +419,14 @@ impl Cache { let progress = self.progress.clone(); let code = code.as_ref().to_vec(); let dir = self.dir.clone(); + let store = self.store.clone(); std::thread::spawn(move || { tracing::info!("Compiling WASM {}.", hash.to_string()); - let (_module, _store) = - match wasm::run::prepare_wasm_code(code) { - Ok(code) => match compile(code) { - Ok((module, store)) => { + let _module = match wasm::run::prepare_wasm_code(code) { + Ok(code) => { + match compile(code, &store) { + Ok(module) => { // Write the file file_write_module(&dir, &module, &hash); @@ -441,7 +451,7 @@ impl Cache { N::name() ) } - (module, store) + module } Err(err) => { let mut progress = @@ -454,19 +464,19 @@ impl Cache { progress.swap_remove(&hash); return Err(err); } - }, - Err(err) => { - let mut progress = - progress.write().unwrap(); - tracing::info!( - "Failed to prepare WASM {} with {}", - hash.to_string(), - err - ); - progress.swap_remove(&hash); - return Err(err); } - }; + } + Err(err) => { + let mut progress = progress.write().unwrap(); + tracing::info!( + "Failed to prepare WASM {} with {}", + hash.to_string(), + err + ); + progress.swap_remove(&hash); + return Err(err); + } + }; let res: Result<(), wasm::run::Error> = Ok(()); res @@ -484,6 +494,7 @@ impl Cache { in_memory: self.in_memory.clone(), name: Default::default(), access: Default::default(), + store: self.store.clone(), } } } @@ -494,8 +505,9 @@ fn hash_of_code(code: impl AsRef<[u8]>) -> Hash { fn compile( code: impl AsRef<[u8]>, -) -> Result<(Module, Store), wasm::run::Error> { - universal::compile(code).map_err(wasm::run::Error::CompileError) + store: &Store, +) -> Result { + universal::compile(code, store).map_err(wasm::run::Error::CompileError) } fn file_ext() -> &'static str { @@ -519,19 +531,19 @@ fn file_write_module(dir: impl AsRef, module: &Module, hash: &Hash) { fn file_load_module( dir: impl AsRef, hash: &Hash, -) -> Result<(Module, Store), wasmer::DeserializeError> { + store: &Store, +) -> Result { use wasmer_cache::Cache; let fs_cache = fs_cache(dir, hash); - let store = store(); let hash = CacheHash::new(hash.0); - let module = unsafe { fs_cache.load(&store, hash) }; + let module = unsafe { fs_cache.load(store, hash) }; if let Err(err) = module.as_ref() { tracing::error!( "Error loading cached wasm {}: {err}.", hash.to_string() ); } - Ok((module?, store)) + module } fn fs_cache(dir: impl AsRef, hash: &Hash) -> FileSystemCache { @@ -564,10 +576,9 @@ mod universal { #[allow(dead_code)] pub fn compile( code: impl AsRef<[u8]>, - ) -> Result<(Module, Store), wasmer::CompileError> { - let store = store(); - let module = Module::new(&store, code.as_ref())?; - Ok((module, store)) + store: &Store, + ) -> Result { + Module::new(store, code.as_ref()) } /// Universal WASM store @@ -585,8 +596,14 @@ pub mod testing { use super::*; use crate::vm::WasmCacheRwAccess; + /// Instantiate the default wasmer store. + pub fn store() -> Store { + super::store() + } + /// Cache with a temp dir for testing - pub fn cache() -> (Cache, TempDir) { + pub fn cache() -> (Cache, TempDir) + { let dir = tempdir().unwrap(); let cache = Cache::new( dir.path(), diff --git a/crates/namada/src/vm/wasm/run.rs b/crates/namada/src/vm/wasm/run.rs index 05acbe2610..dea4a1cbad 100644 --- a/crates/namada/src/vm/wasm/run.rs +++ b/crates/namada/src/vm/wasm/run.rs @@ -282,6 +282,9 @@ where _ = (instance, env); if ok == 1 { + let store = Rc::into_inner(store) + .expect("The store must be dropped after execution to avoid leaks"); + let _store = RefCell::into_inner(store); Ok(verifiers) } else { let err = yielded_value.take().map_or_else( @@ -490,6 +493,9 @@ where _ = (instance, vp_imports); if is_valid == 1 { + let store = Rc::into_inner(store) + .expect("The store must be dropped after execution to avoid leaks"); + let _store = RefCell::into_inner(store); Ok(()) } else { unsafe { yielded_value.get_mut() }.take().map_or_else( @@ -936,6 +942,7 @@ impl wasm_instrument::gas_metering::Rules for GasRules { #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::error::Error as StdErrorTrait; use borsh_ext::BorshSerializeExt; @@ -985,7 +992,7 @@ mod tests { const PANIC_MSG: &str = "Test should have failed with a wasm runtime memory error"; - let error = execute_tx_with_code(tx_code).expect_err(PANIC_MSG); + let error = execute_tx_with_code(&tx_code).expect_err(PANIC_MSG); assert!( matches!( assert_tx_rt_mem_error(&error, PANIC_MSG), @@ -1904,7 +1911,7 @@ mod tests { const PANIC_MSG: &str = "Test should have failed with a wasm runtime memory error"; - let error = execute_tx_with_code(tx_code).expect_err(PANIC_MSG); + let error = execute_tx_with_code(&tx_code).expect_err(PANIC_MSG); assert!( matches!( assert_tx_rt_mem_error(&error, PANIC_MSG), @@ -1944,7 +1951,7 @@ mod tests { const PANIC_MSG: &str = "Test should have failed with a wasm runtime memory error"; - let error = execute_vp_with_code(vp_code).expect_err(PANIC_MSG); + let error = execute_vp_with_code(&vp_code).expect_err(PANIC_MSG); assert!( matches!( assert_vp_rt_mem_error(&error, PANIC_MSG), @@ -1954,7 +1961,71 @@ mod tests { ); } - fn execute_vp_with_code(vp_code: Vec) -> Result<()> { + #[test] + fn test_tx_leak() { + let tx_code = TestWasms::TxNoOp.read_bytes(); + let (mut vp_cache, _) = + wasm::compilation_cache::common::testing::cache(); + let (mut tx_cache, _) = + wasm::compilation_cache::common::testing::cache(); + let mut last_cache_size: Option = None; + for _ in 0..3 { + let _verifiers = execute_tx_with_code_and_cache( + &tx_code, + &mut tx_cache, + &mut vp_cache, + ) + .unwrap(); + + let info = &wasmer_compiler::FRAME_INFO.read().unwrap(); + let info: &GlobalFrameInfo = unsafe { std::mem::transmute(info) }; + if let Some(last_cache_size) = last_cache_size { + assert_eq!( + last_cache_size, + info.ranges.len(), + "The frame info must not be growing - we're using the \ + same WASM in each loop" + ); + } else { + last_cache_size = Some(info.ranges.len()); + } + } + } + + #[test] + fn test_vp_leak() { + let vp_code = TestWasms::VpAlwaysTrue.read_bytes(); + let (mut vp_cache, _) = + wasm::compilation_cache::common::testing::cache(); + let mut last_cache_size: Option = None; + for _ in 0..3 { + execute_vp_with_code_and_cache(&vp_code, &mut vp_cache).unwrap(); + + let info = &wasmer_compiler::FRAME_INFO.read().unwrap(); + let info: &GlobalFrameInfo = unsafe { std::mem::transmute(info) }; + if let Some(last_cache_size) = last_cache_size { + assert_eq!( + last_cache_size, + info.ranges.len(), + "The frame info must not be growing - we're using the \ + same WASM in each loop" + ); + } else { + last_cache_size = Some(info.ranges.len()); + } + } + } + + fn execute_vp_with_code(vp_code: &[u8]) -> Result<()> { + let (mut vp_cache, _) = + wasm::compilation_cache::common::testing::cache(); + execute_vp_with_code_and_cache(vp_code, &mut vp_cache) + } + + fn execute_vp_with_code_and_cache( + vp_code: &[u8], + vp_cache: &mut VpCache, + ) -> Result<()> { let mut outer_tx = Tx::from_type(TxType::Raw); outer_tx.push_default_inner_tx(); let tx_index = TxIndex::default(); @@ -1965,9 +2036,8 @@ mod tests { )); let keys_changed = BTreeSet::new(); let verifiers = BTreeSet::new(); - let (vp_cache, _) = wasm::compilation_cache::common::testing::cache(); // store the vp code - let code_hash = Hash::sha256(&vp_code); + let code_hash = Hash::sha256(vp_code); let code_len = vp_code.len() as u64; let key = Key::wasm_code(&code_hash); let len_key = Key::wasm_code_len(&code_hash); @@ -1983,23 +2053,31 @@ mod tests { &gas_meter, &keys_changed, &verifiers, - vp_cache, + vp_cache.clone(), ) } - fn execute_tx_with_code(tx_code: Vec) -> Result> { + fn execute_tx_with_code(tx_code: &[u8]) -> Result> { + let (mut tx_cache, _) = + wasm::compilation_cache::common::testing::cache(); + let (mut vp_cache, _) = + wasm::compilation_cache::common::testing::cache(); + execute_tx_with_code_and_cache(tx_code, &mut tx_cache, &mut vp_cache) + } + + fn execute_tx_with_code_and_cache( + tx_code: &[u8], + tx_cache: &mut TxCache, + vp_cache: &mut VpCache, + ) -> Result> { let tx_data = vec![]; let tx_index = TxIndex::default(); let mut state = TestState::default(); let gas_meter = RefCell::new(TxGasMeter::new_from_sub_limit(TX_GAS_LIMIT.into())); - let (mut vp_cache, _) = - wasm::compilation_cache::common::testing::cache(); - let (mut tx_cache, _) = - wasm::compilation_cache::common::testing::cache(); // store the tx code - let code_hash = Hash::sha256(&tx_code); + let code_hash = Hash::sha256(tx_code); let code_len = (tx_code.len() as u64).serialize_to_vec(); let key = Key::wasm_code(&code_hash); let len_key = Key::wasm_code_len(&code_hash); @@ -2020,8 +2098,8 @@ mod tests { &tx_index, batched_tx.tx, batched_tx.cmt, - &mut vp_cache, - &mut tx_cache, + vp_cache, + tx_cache, ) } @@ -2058,7 +2136,7 @@ mod tests { .expect("unexpected error converting wat2wasm") .into_owned(); - execute_tx_with_code(tx_code) + execute_tx_with_code(&tx_code) } fn loop_in_vp_wasm(loops: u32) -> Result<()> { @@ -2090,7 +2168,7 @@ mod tests { ) .expect("unexpected error converting wat2wasm").into_owned(); - execute_vp_with_code(vp_code) + execute_vp_with_code(&vp_code) } fn get_trap_code(error: &Error) -> Either { @@ -2117,4 +2195,22 @@ mod tests { Either::Left(wasmer_vm::TrapCode::StackOverflow), ); } + + /// The following definitions are copied from wasmer v4.3.5 + /// `lib/compiler/src/engine/trap/frame_info.rs` to access internal + /// fields that are otherwise private. This must be carefully maintained + /// while we workaround the leak before it's fixed in wasmer. + pub struct GlobalFrameInfo { + ranges: BTreeMap, + } + struct ModuleInfoFrameInfo { + _start: usize, + _functions: BTreeMap, + _module: std::sync::Arc, + _frame_infos: wasmer_compiler::FrameInfosVariant, + } + struct FunctionInfo { + _start: usize, + _local_index: wasmer_types::LocalFunctionIndex, + } } diff --git a/crates/tests/Cargo.toml b/crates/tests/Cargo.toml index 6c76c5d816..3534c4eaa2 100644 --- a/crates/tests/Cargo.toml +++ b/crates/tests/Cargo.toml @@ -17,7 +17,7 @@ default = ["wasm-runtime", "migrations"] mainnet = [ "namada/mainnet", ] -wasm-runtime = ["namada/wasm-runtime"] +wasm-runtime = ["namada/wasm-runtime", "wasmer"] integration = ["namada_node/integration", "namada_apps_lib/integration"] migrations = [ "namada/migrations", @@ -52,6 +52,7 @@ sha2.workspace = true tempfile.workspace = true test-log.workspace = true tokio = {workspace = true, features = ["full"]} +wasmer = { workspace = true, optional = true } [dev-dependencies] namada_apps_lib = {path = "../apps_lib", features = ["testing"]} diff --git a/crates/tests/src/vm_host_env/tx.rs b/crates/tests/src/vm_host_env/tx.rs index 801d5d7c51..0abd79bd5f 100644 --- a/crates/tests/src/vm_host_env/tx.rs +++ b/crates/tests/src/vm_host_env/tx.rs @@ -1,6 +1,7 @@ use std::borrow::Borrow; use std::cell::RefCell; use std::collections::BTreeSet; +use std::rc::Rc; use namada::core::address::Address; use namada::core::hash::Hash; @@ -59,6 +60,7 @@ pub struct TestTxEnv { pub tx_wasm_cache: TxCache, pub tx_cache_dir: TempDir, pub batched_tx: BatchedTx, + pub wasmer_store: Rc>, } impl Default for TestTxEnv { fn default() -> Self { @@ -71,6 +73,11 @@ impl Default for TestTxEnv { tx.header.chain_id = state.in_mem().chain_id.clone(); tx.push_default_inner_tx(); let batched_tx = tx.batch_first_tx(); + + let wasmer_store = Rc::new(RefCell::new( + wasm::compilation_cache::common::testing::store(), + )); + Self { state, iterators: PrefixIterators::default(), @@ -87,6 +94,7 @@ impl Default for TestTxEnv { tx_wasm_cache, tx_cache_dir, batched_tx, + wasmer_store, } } } @@ -346,6 +354,7 @@ mod native_tx_host_env { tx_wasm_cache, tx_cache_dir: _, batched_tx, + wasmer_store: _, }: &mut TestTxEnv| { let mut tx_env = vm::host_env::testing::tx_env( @@ -377,7 +386,7 @@ mod native_tx_host_env { #[no_mangle] extern "C" fn extern_fn_name( $($arg: $type),* ) -> $ret { with(|TestTxEnv { - tx_index, + tx_index, state, iterators, verifiers, @@ -389,7 +398,8 @@ mod native_tx_host_env { vp_cache_dir: _, tx_wasm_cache, tx_cache_dir: _, - batched_tx + batched_tx, + wasmer_store: _, }: &mut TestTxEnv| { let mut tx_env = vm::host_env::testing::tx_env( @@ -434,6 +444,7 @@ mod native_tx_host_env { tx_wasm_cache, tx_cache_dir: _, batched_tx, + wasmer_store: _, }: &mut TestTxEnv| { let mut tx_env = vm::host_env::testing::tx_env( @@ -749,6 +760,7 @@ mod tests { tx_wasm_cache, tx_cache_dir: _, batched_tx, + wasmer_store, } = test_env; let mut tx_env = vm::host_env::testing::tx_env_with_wasm_memory( @@ -762,6 +774,7 @@ mod tests { tx_index, result_buffer, yielded_value, + wasmer_store.clone(), vp_wasm_cache, tx_wasm_cache, ); diff --git a/wasm/Cargo.lock b/wasm/Cargo.lock index 94a4eac8da..c6da414347 100644 --- a/wasm/Cargo.lock +++ b/wasm/Cargo.lock @@ -3507,10 +3507,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] @@ -4173,6 +4173,7 @@ dependencies = [ "tempfile", "test-log", "tokio", + "wasmer", ] [[package]] @@ -5229,14 +5230,14 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "region" -version = "3.0.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e189c2369884dce920945e2ddf79b3dff49e071a167dd1817fa9c4c00d512e" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" dependencies = [ "bitflags 1.3.2", "libc", - "mach", - "winapi", + "mach2", + "windows-sys 0.52.0", ] [[package]] @@ -7330,9 +7331,9 @@ dependencies = [ [[package]] name = "wasmer" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce4a267a570e121c9375136adefa2c48810273907de9c6817bc19db4d6144bc" +checksum = "c3a6e0f73e5ae361fe64db607eaf4ab2381d88ad2c1b0bb8cf254cf35d894687" dependencies = [ "bytes", "cfg-if", @@ -7359,9 +7360,9 @@ dependencies = [ [[package]] name = "wasmer-cache" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a40804bcc2567f112003182fc5edc29584da5199c4a1f5a8d6a6e4b65feff0" +checksum = "79fd0889f8844b7c70b8ee8fbf1d1f6ccff99399c6f3d3627048cde04b1ac493" dependencies = [ "blake3", "hex", @@ -7371,9 +7372,9 @@ dependencies = [ [[package]] name = "wasmer-compiler" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9c23098e86ef1038155684fe50f0c1079a0e2a2e70f115b789df17e6ba98d20" +checksum = "cb1e7c79507f5d55f1afd99984717e8380440cd98e13d542e4d00661f986f2d4" dependencies = [ "backtrace", "bytes", @@ -7382,6 +7383,7 @@ dependencies = [ "enumset", "lazy_static", "leb128", + "libc", "memmap2 0.5.10", "more-asserts", "region", @@ -7399,9 +7401,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-cranelift" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95287b79973ad5f485215733ef9f0d4bb951a6b7e655585d2bd3d4a4ba1253c9" +checksum = "8f3352014573750327646a690d32774312b0e8b7920e7e8ba00c0449eac18390" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -7418,9 +7420,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-singlepass" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00d78d59be3ce78ad859e176b88f0d5bec0120ece0684922d7c5da1289e251b1" +checksum = "cc490f011e855eb5e31475ff3e9efa83840cc0ed2f8322dfaca18627df0a9f3c" dependencies = [ "byteorder", "dynasm", @@ -7459,9 +7461,9 @@ dependencies = [ [[package]] name = "wasmer-derive" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48f36aeeecb655f15fdd358bdf6e4cec27df181468fa4226084157e8462bd5e" +checksum = "ac6b0b0580cfa1fc7ad58cca3626a742f2b2e5ccd51cfc5de43e8edb0d1daa4c" dependencies = [ "proc-macro-error", "proc-macro2", @@ -7471,9 +7473,9 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83cb97b6b20084757a2a8d548dc0d4179c3fe9e2d711740423a1e6aa3f8b9091" +checksum = "576442cc3d302ca215fd40aa7826a078571dca7eaa773d8cdedca14a2ec7c9a1" dependencies = [ "bytecheck", "enum-iterator", @@ -7492,9 +7494,9 @@ dependencies = [ [[package]] name = "wasmer-vm" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1e19d986844b17b927ec8b0c7f3da6a7a2c2cb3b0f8ca5d4cb1a1f71bfb124" +checksum = "6483035d1df84a978cd6c6a35878e913dc8ec6311f8712548a922a75e87957ba" dependencies = [ "backtrace", "cc", @@ -7508,7 +7510,7 @@ dependencies = [ "indexmap 1.9.3", "lazy_static", "libc", - "mach", + "mach2", "memoffset", "more-asserts", "region", diff --git a/wasm_for_tests/Cargo.lock b/wasm_for_tests/Cargo.lock index 2e7778bbe4..7813ff2a03 100644 --- a/wasm_for_tests/Cargo.lock +++ b/wasm_for_tests/Cargo.lock @@ -83,55 +83,12 @@ dependencies = [ "libc", ] -[[package]] -name = "anstream" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - [[package]] name = "anstyle" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" -[[package]] -name = "anstyle-parse" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - [[package]] name = "anyhow" version = "1.0.75" @@ -846,7 +803,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" dependencies = [ "clap_builder", - "clap_derive", ] [[package]] @@ -855,22 +811,8 @@ version = "4.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ - "anstream", "anstyle", "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.65", ] [[package]] @@ -936,12 +878,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "colorchoice" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" - [[package]] name = "concat-idents" version = "1.1.5" @@ -3382,12 +3318,6 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" -[[package]] -name = "is_terminal_polyfill" -version = "1.70.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" - [[package]] name = "itertools" version = "0.11.0" @@ -3557,10 +3487,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] -name = "mach" -version = "0.3.2" +name = "mach2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" dependencies = [ "libc", ] @@ -4192,6 +4122,7 @@ dependencies = [ "tempfile", "test-log", "tokio", + "wasmer", ] [[package]] @@ -5236,14 +5167,14 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "region" -version = "3.0.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e189c2369884dce920945e2ddf79b3dff49e071a167dd1817fa9c4c00d512e" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" dependencies = [ "bitflags 1.3.2", "libc", - "mach", - "winapi", + "mach2", + "windows-sys 0.52.0", ] [[package]] @@ -7062,12 +6993,6 @@ dependencies = [ "serde", ] -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - [[package]] name = "uuid" version = "0.8.2" @@ -7316,9 +7241,9 @@ dependencies = [ [[package]] name = "wasmer" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce4a267a570e121c9375136adefa2c48810273907de9c6817bc19db4d6144bc" +checksum = "c3a6e0f73e5ae361fe64db607eaf4ab2381d88ad2c1b0bb8cf254cf35d894687" dependencies = [ "bytes", "cfg-if", @@ -7345,9 +7270,9 @@ dependencies = [ [[package]] name = "wasmer-cache" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a40804bcc2567f112003182fc5edc29584da5199c4a1f5a8d6a6e4b65feff0" +checksum = "79fd0889f8844b7c70b8ee8fbf1d1f6ccff99399c6f3d3627048cde04b1ac493" dependencies = [ "blake3", "hex", @@ -7357,9 +7282,9 @@ dependencies = [ [[package]] name = "wasmer-compiler" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9c23098e86ef1038155684fe50f0c1079a0e2a2e70f115b789df17e6ba98d20" +checksum = "cb1e7c79507f5d55f1afd99984717e8380440cd98e13d542e4d00661f986f2d4" dependencies = [ "backtrace", "bytes", @@ -7368,6 +7293,7 @@ dependencies = [ "enumset", "lazy_static", "leb128", + "libc", "memmap2 0.5.10", "more-asserts", "region", @@ -7385,9 +7311,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-cranelift" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95287b79973ad5f485215733ef9f0d4bb951a6b7e655585d2bd3d4a4ba1253c9" +checksum = "8f3352014573750327646a690d32774312b0e8b7920e7e8ba00c0449eac18390" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -7404,9 +7330,9 @@ dependencies = [ [[package]] name = "wasmer-compiler-singlepass" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00d78d59be3ce78ad859e176b88f0d5bec0120ece0684922d7c5da1289e251b1" +checksum = "cc490f011e855eb5e31475ff3e9efa83840cc0ed2f8322dfaca18627df0a9f3c" dependencies = [ "byteorder", "dynasm", @@ -7445,9 +7371,9 @@ dependencies = [ [[package]] name = "wasmer-derive" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48f36aeeecb655f15fdd358bdf6e4cec27df181468fa4226084157e8462bd5e" +checksum = "ac6b0b0580cfa1fc7ad58cca3626a742f2b2e5ccd51cfc5de43e8edb0d1daa4c" dependencies = [ "proc-macro-error", "proc-macro2", @@ -7457,9 +7383,9 @@ dependencies = [ [[package]] name = "wasmer-types" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83cb97b6b20084757a2a8d548dc0d4179c3fe9e2d711740423a1e6aa3f8b9091" +checksum = "576442cc3d302ca215fd40aa7826a078571dca7eaa773d8cdedca14a2ec7c9a1" dependencies = [ "bytecheck", "enum-iterator", @@ -7478,9 +7404,9 @@ dependencies = [ [[package]] name = "wasmer-vm" -version = "4.3.1" +version = "4.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1e19d986844b17b927ec8b0c7f3da6a7a2c2cb3b0f8ca5d4cb1a1f71bfb124" +checksum = "6483035d1df84a978cd6c6a35878e913dc8ec6311f8712548a922a75e87957ba" dependencies = [ "backtrace", "cc", @@ -7494,7 +7420,7 @@ dependencies = [ "indexmap 1.9.3", "lazy_static", "libc", - "mach", + "mach2", "memoffset", "more-asserts", "region", @@ -7572,15 +7498,14 @@ dependencies = [ [[package]] name = "webc" -version = "6.0.0-alpha9" +version = "6.0.0-rc1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b4e8dd987046eede4348d660404ff990412631b7d493f9e547adcf2862cd5" +checksum = "c1fc686c7b43c9bc630a499f6ae1f0a4c4bd656576a53ae8a147b0cc9bc983ad" dependencies = [ "anyhow", "base64 0.21.7", "bytes", "cfg-if", - "clap", "document-features", "flate2", "indexmap 1.9.3",