Add support for async ABI, futures, streams, and errors (#1895)

* Add support for async ABI, futures, streams, and errors This adds support for encoding and parsing components which use the [Async ABI](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Async.md) and associated canonical options and functions, along with the [`stream`, `future`, and `error`](WebAssembly/component-model#405) types. See bytecodealliance/rfcs#38 for more context. Signed-off-by: Joel Dice <[email protected]> add wasmparser::WasmFeatures support to wasm-compose Signed-off-by: Joel Dice <[email protected]> fix no-std build in readers.rs Signed-off-by: Joel Dice <[email protected]> rename `error` to `error-context` per latest spec Signed-off-by: Joel Dice <[email protected]> rename `error` to `error-context` per latest spec (part 2) Also, parse string encoding and realloc from encoded `error-context.new` and `error-context.debug-string` names. Signed-off-by: Joel Dice <[email protected]> add `wast` support for parsing async canon opts And add tests/local/component-model-async/lift-async.wast for round-trip testing of async lifts (more to come). Signed-off-by: Joel Dice <[email protected]> more wast async support and more tests This also fixes a bug in `wasmprinter` keeping track of core functions. Signed-off-by: Joel Dice <[email protected]> more wast async support; add async tests; fix bugs Signed-off-by: Joel Dice <[email protected]> more component-model-async tests and fixes Signed-off-by: Joel Dice <[email protected]> add `wit-parser` tests for streams, futures, and error-contexts Signed-off-by: Joel Dice <[email protected]> add first `wit-component` async test This required adding a new `wit_parser::decoding::decode_reader_with_features` function for passing `WasmFeatures` to `wasmparser::Validator::new_with_features`. Signed-off-by: Joel Dice <[email protected]> add more async tests Signed-off-by: Joel Dice <[email protected]> add `async-builtins` test for `wit-component` Signed-off-by: Joel Dice <[email protected]> add `async-streams-and-futures` test to `wit-component` Signed-off-by: Joel Dice <[email protected]> fix stream/future type handling for exported interfaces Signed-off-by: Joel Dice <[email protected]> support callback-less (AKA stackful) async lifts Signed-off-by: Joel Dice <[email protected]> address review feedback Signed-off-by: Joel Dice <[email protected]> * address more review feedback Signed-off-by: Joel Dice <[email protected]> * fix test build regression Signed-off-by: Joel Dice <[email protected]> * fix more test regressions Signed-off-by: Joel Dice <[email protected]> * remove need for `Default` impls in `wast::component::func` Signed-off-by: Joel Dice <[email protected]> * fix another test regression Signed-off-by: Joel Dice <[email protected]> * minor code simplification Signed-off-by: Joel Dice <[email protected]> * require wasmparser `features` feature in wasm-compose This fixes `cargo check --no-default-features --feature compose`. Signed-off-by: Joel Dice <[email protected]> * require wasmparser `features` feature in wit-parser Signed-off-by: Joel Dice <[email protected]> * simplify `r#async` parsing code Signed-off-by: Joel Dice <[email protected]> * add a bunch of comments to wit-component async code Signed-off-by: Joel Dice <[email protected]> * allow importing the same function sync and async Signed-off-by: Joel Dice <[email protected]> --------- Signed-off-by: Joel Dice <[email protected]>
bytecodealliance · Dec 16, 2024 · 9d30160 · 9d30160
1 parent 4d785bb
commit 9d30160
Show file tree

Hide file tree

Showing 115 changed files with 8,579 additions and 439 deletions.
diff --git a/crates/wasm-compose/Cargo.toml b/crates/wasm-compose/Cargo.toml
@@ -17,7 +17,7 @@ workspace = true
 [dependencies]
 wat = { workspace = true }
 wasm-encoder = { workspace = true, features = ['wasmparser', 'component-model'] }
-wasmparser = { workspace = true, features = ['validate', 'component-model'] }
+wasmparser = { workspace = true, features = ['validate', 'component-model', 'features'] }
 indexmap = { workspace = true, features = ["serde"] }
 anyhow = { workspace = true }
 serde = { workspace = true }

diff --git a/crates/wasm-compose/src/composer.rs b/crates/wasm-compose/src/composer.rs
@@ -497,8 +497,6 @@ impl<'a> CompositionGraphBuilder<'a> {
             }
         }
 
-        self.graph.unify_imported_resources();
-
         Ok((self.instances[root_instance], self.graph))
     }
 }

diff --git a/crates/wasm-compose/src/encoding.rs b/crates/wasm-compose/src/encoding.rs
@@ -596,6 +596,17 @@ impl<'a> TypeEncoder<'a> {
                 return ret;
             }
 
+            if let Some((instance, name)) = state.cur.instance_exports.get(&key) {
+                let ret = state.cur.encodable.type_count();
+                state.cur.encodable.alias(Alias::InstanceExport {
+                    instance: *instance,
+                    name,
+                    kind: ComponentExportKind::Type,
+                });
+                log::trace!("id defined in current instance");
+                return ret;
+            }
+
             match id.peel_alias(&self.0.types) {
                 Some(next) => id = next,
                 // If there's no more aliases then fall through to the
@@ -608,15 +619,17 @@ impl<'a> TypeEncoder<'a> {
         return match id {
             AnyTypeId::Core(ComponentCoreTypeId::Sub(_)) => unreachable!(),
             AnyTypeId::Core(ComponentCoreTypeId::Module(id)) => self.module_type(state, id),
-            AnyTypeId::Component(id) => match id {
-                ComponentAnyTypeId::Resource(_) => {
-                    unreachable!("should have been handled in `TypeEncoder::component_entity_type`")
+            AnyTypeId::Component(id) => {
+                match id {
+                    ComponentAnyTypeId::Resource(r) => {
+                        unreachable!("should have been handled in `TypeEncoder::component_entity_type`: {r:?}")
+                    }
+                    ComponentAnyTypeId::Defined(id) => self.defined_type(state, id),
+                    ComponentAnyTypeId::Func(id) => self.component_func_type(state, id),
+                    ComponentAnyTypeId::Instance(id) => self.component_instance_type(state, id),
+                    ComponentAnyTypeId::Component(id) => self.component_type(state, id),
                 }
-                ComponentAnyTypeId::Defined(id) => self.defined_type(state, id),
-                ComponentAnyTypeId::Func(id) => self.component_func_type(state, id),
-                ComponentAnyTypeId::Instance(id) => self.component_instance_type(state, id),
-                ComponentAnyTypeId::Component(id) => self.component_type(state, id),
-            },
+            }
         };
     }
 
@@ -667,6 +680,9 @@ impl<'a> TypeEncoder<'a> {
                 state.cur.encodable.ty().defined_type().borrow(ty);
                 index
             }
+            ComponentDefinedType::Future(ty) => self.future(state, *ty),
+            ComponentDefinedType::Stream(ty) => self.stream(state, *ty),
+            ComponentDefinedType::ErrorContext => self.error_context(state),
         }
     }
 
@@ -788,6 +804,28 @@ impl<'a> TypeEncoder<'a> {
         }
         export
     }
+
+    fn future(&self, state: &mut TypeState<'a>, ty: Option<ct::ComponentValType>) -> u32 {
+        let ty = ty.map(|ty| self.component_val_type(state, ty));
+
+        let index = state.cur.encodable.type_count();
+        state.cur.encodable.ty().defined_type().future(ty);
+        index
+    }
+
+    fn stream(&self, state: &mut TypeState<'a>, ty: ct::ComponentValType) -> u32 {
+        let ty = self.component_val_type(state, ty);
+
+        let index = state.cur.encodable.type_count();
+        state.cur.encodable.ty().defined_type().stream(ty);
+        index
+    }
+
+    fn error_context(&self, state: &mut TypeState<'a>) -> u32 {
+        let index = state.cur.encodable.type_count();
+        state.cur.encodable.ty().defined_type().error_context();
+        index
+    }
 }
 
 /// Represents an instance index in a composition graph.
@@ -1215,8 +1253,11 @@ impl DependencyRegistrar<'_, '_> {
         match &self.types[ty] {
             ComponentDefinedType::Primitive(_)
             | ComponentDefinedType::Enum(_)
-            | ComponentDefinedType::Flags(_) => {}
-            ComponentDefinedType::List(t) | ComponentDefinedType::Option(t) => self.val_type(*t),
+            | ComponentDefinedType::Flags(_)
+            | ComponentDefinedType::ErrorContext => {}
+            ComponentDefinedType::List(t)
+            | ComponentDefinedType::Option(t)
+            | ComponentDefinedType::Stream(t) => self.val_type(*t),
             ComponentDefinedType::Own(r) | ComponentDefinedType::Borrow(r) => {
                 self.ty(ComponentAnyTypeId::Resource(*r))
             }
@@ -1245,6 +1286,11 @@ impl DependencyRegistrar<'_, '_> {
                     self.val_type(*err);
                 }
             }
+            ComponentDefinedType::Future(ty) => {
+                if let Some(ty) = ty {
+                    self.val_type(*ty);
+                }
+            }
         }
     }
 }
@@ -1402,7 +1448,7 @@ impl<'a> CompositionGraphEncoder<'a> {
         state.push(Encodable::Instance(InstanceType::new()));
         for (name, types) in exports {
             let (component, ty) = types[0];
-            log::trace!("export {name}");
+            log::trace!("export {name}: {ty:?}");
             let export = TypeEncoder::new(component).export(name, ty, state);
             let t = match &mut state.cur.encodable {
                 Encodable::Instance(c) => c,
@@ -1418,6 +1464,7 @@ impl<'a> CompositionGraphEncoder<'a> {
                 }
             }
         }
+
         let instance_type = match state.pop() {
             Encodable::Instance(c) => c,
             _ => unreachable!(),

diff --git a/crates/wasm-compose/src/graph.rs b/crates/wasm-compose/src/graph.rs
@@ -18,7 +18,7 @@ use wasmparser::{
     names::ComponentName,
     types::{Types, TypesRef},
     Chunk, ComponentExternalKind, ComponentTypeRef, Encoding, Parser, Payload, ValidPayload,
-    Validator,
+    Validator, WasmFeatures,
 };
 
 pub(crate) fn type_desc(item: ComponentEntityType) -> &'static str {
@@ -99,7 +99,7 @@ impl<'a> Component<'a> {
     fn parse(name: String, path: Option<PathBuf>, bytes: Cow<'a, [u8]>) -> Result<Self> {
         let mut parser = Parser::new(0);
         let mut parsers = Vec::new();
-        let mut validator = Validator::new();
+        let mut validator = Validator::new_with_features(WasmFeatures::all());
         let mut imports = IndexMap::new();
         let mut exports = IndexMap::new();
 
@@ -439,7 +439,7 @@ pub(crate) struct Instance {
 }
 
 /// The options for encoding a composition graph.
-#[derive(Debug, Clone, Copy, Eq, PartialEq, Default)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Default)]
 pub struct EncodeOptions {
     /// Whether or not to define instantiated components.
     ///
@@ -448,7 +448,7 @@ pub struct EncodeOptions {
 
     /// The instance in the graph to export.
     ///
-    /// If `Some`, the instance's exports will be aliased and
+    /// If non-empty, the instance's exports will be aliased and
     /// exported from the resulting component.
     pub export: Option<InstanceId>,
 
@@ -508,9 +508,6 @@ impl ResourceMapping {
                         if value.1 == export_resource {
                             self.map.insert(export_resource, value);
                             self.map.insert(import_resource, value);
-                        } else {
-                            // Can't set two different exports equal to each other -- give up.
-                            return None;
                         }
                     } else {
                         // Couldn't find an export with a name that matches this
@@ -559,14 +556,19 @@ impl<'a> CompositionGraph<'a> {
     /// connected to exports, group them by name, and update the resource
     /// mapping to make all resources within each group equivalent.
     ///
-    /// This should be the last step prior to encoding, after all
-    /// inter-component connections have been made.  It ensures that each set of
-    /// identical imports composed component can be merged into a single import
-    /// in the output component.
+    /// This ensures that each set of identical imports in the composed
+    /// components can be merged into a single import in the output component.
+    //
+    // TODO: How do we balance the need to call this early (so we can match up
+    // imports with exports which mutually import the same resources) with the
+    // need to delay decisions about where resources are coming from (so that we
+    // can match up imported resources with exported resources)?  Right now I
+    // think we're erring on the side if the former at the expense of the
+    // latter.
     pub(crate) fn unify_imported_resources(&self) {
         let mut resource_mapping = self.resource_mapping.borrow_mut();
 
-        let mut resource_imports = HashMap::<_, Vec<_>>::new();
+        let mut resource_imports = IndexMap::<_, IndexSet<_>>::new();
         for (component_id, component) in &self.components {
             let component = &component.component;
             for import_name in component.imports.keys() {
@@ -584,20 +586,22 @@ impl<'a> CompositionGraph<'a> {
                             ..
                         } = ty
                         {
-                            if !resource_mapping.map.contains_key(&resource_id.resource()) {
-                                resource_imports
-                                    .entry(vec![import_name.to_string(), export_name.to_string()])
-                                    .or_default()
-                                    .push((*component_id, resource_id.resource()))
+                            let set = resource_imports
+                                .entry(vec![import_name.to_string(), export_name.to_string()])
+                                .or_default();
+
+                            if let Some(pair) = resource_mapping.map.get(&resource_id.resource()) {
+                                set.insert(*pair);
                             }
+                            set.insert((*component_id, resource_id.resource()));
                         }
                     }
                 }
             }
         }
 
         for resources in resource_imports.values() {
-            match &resources[..] {
+            match &resources.iter().copied().collect::<Vec<_>>()[..] {
                 [] => unreachable!(),
                 [_] => {}
                 [first, rest @ ..] => {
@@ -653,10 +657,8 @@ impl<'a> CompositionGraph<'a> {
                 .remap_component_entity(&mut import_type, remapping);
             remapping.reset_type_cache();
 
-            if context
-                .component_entity_type(&export_type, &import_type, 0)
-                .is_ok()
-            {
+            let v = context.component_entity_type(&export_type, &import_type, 0);
+            if v.is_ok() {
                 *self.resource_mapping.borrow_mut() = resource_mapping;
                 true
             } else {
@@ -706,6 +708,10 @@ impl<'a> CompositionGraph<'a> {
 
         assert!(self.components.insert(id, entry).is_none());
 
+        if self.components.len() > 1 {
+            self.unify_imported_resources();
+        }
+
         Ok(id)
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -497,8 +497,6 @@ impl<'a> CompositionGraphBuilder<'a> { @@
                 }
             }
-            self.graph.unify_imported_resources();
             Ok((self.instances[root_instance], self.graph))
         }
     }
@@ Expand Down @@