Skip to content

Commit

Permalink
Add support for async ABI, futures, streams, and errors
Browse files Browse the repository at this point in the history 
This adds support for encoding and parsing components which use the [Async
ABI](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Async.md)
and associated canonical options and functions, along with the [`stream`,
`future`, and `error`](WebAssembly/component-model#405)
types.

See bytecodealliance/rfcs#38 for more context.

Signed-off-by: Joel Dice <[email protected]>

add wasmparser::WasmFeatures support to wasm-compose

Signed-off-by: Joel Dice <[email protected]>

fix no-std build in readers.rs

Signed-off-by: Joel Dice <[email protected]>

rename `error` to `error-context` per latest spec

Signed-off-by: Joel Dice <[email protected]>

rename `error` to `error-context` per latest spec (part 2)

Also, parse string encoding and realloc from encoded `error-context.new` and
`error-context.debug-string` names.

Signed-off-by: Joel Dice <[email protected]>

add `wast` support for parsing async canon opts

And add tests/local/component-model-async/lift-async.wast for round-trip testing
of async lifts (more to come).

Signed-off-by: Joel Dice <[email protected]>

more wast async support and more tests

This also fixes a bug in `wasmprinter` keeping track of core functions.

Signed-off-by: Joel Dice <[email protected]>

more wast async support; add async tests; fix bugs

Signed-off-by: Joel Dice <[email protected]>

more component-model-async tests and fixes

Signed-off-by: Joel Dice <[email protected]>

add `wit-parser` tests for streams, futures, and error-contexts

Signed-off-by: Joel Dice <[email protected]>

add first `wit-component` async test

This required adding a new `wit_parser::decoding::decode_reader_with_features`
function for passing `WasmFeatures` to
`wasmparser::Validator::new_with_features`.

Signed-off-by: Joel Dice <[email protected]>

add more async tests

Signed-off-by: Joel Dice <[email protected]>

add `async-builtins` test for `wit-component`

Signed-off-by: Joel Dice <[email protected]>

add `async-streams-and-futures` test to `wit-component`

Signed-off-by: Joel Dice <[email protected]>

fix stream/future type handling for exported interfaces

Signed-off-by: Joel Dice <[email protected]>

support callback-less (AKA stackful) async lifts

Signed-off-by: Joel Dice <[email protected]>
  • Loading branch information
@dicej
dicej committed Dec 10, 2024
1 parent 1bcecbc commit c8c3f47
Show file tree
Hide file tree
Showing 116 changed files with 9,169 additions and 352 deletions.
68 changes: 46 additions & 22 deletions crates/wasm-compose/src/composer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use std::{collections::VecDeque, ffi::OsStr, path::Path};
use wasmparser::{
component_types::{ComponentEntityType, ComponentInstanceTypeId},
types::TypesRef,
ComponentExternalKind, ComponentTypeRef,
ComponentExternalKind, ComponentTypeRef, WasmFeatures,
};

/// The root component name used in configuration.
Expand Down Expand Up @@ -72,9 +72,13 @@ struct CompositionGraphBuilder<'a> {
}

impl<'a> CompositionGraphBuilder<'a> {
fn new(root_path: &Path, config: &'a Config) -> Result<Self> {
fn new(root_path: &Path, config: &'a Config, features: WasmFeatures) -> Result<Self> {
let mut graph = CompositionGraph::new();
graph.add_component(Component::from_file(ROOT_COMPONENT_NAME, root_path)?)?;
graph.add_component(Component::from_file(
ROOT_COMPONENT_NAME,
root_path,
features,
)?)?;

let definitions = config
.definitions
Expand All @@ -87,7 +91,7 @@ impl<'a> CompositionGraphBuilder<'a> {
)
})?;

let component = Component::from_file(name, config.dir.join(path))?;
let component = Component::from_file(name, config.dir.join(path), features)?;

Ok((graph.add_component(component)?, None))
})
Expand All @@ -105,19 +109,19 @@ impl<'a> CompositionGraphBuilder<'a> {
///
/// If a component with the given name already exists, its id is returned.
/// Returns `Ok(None)` if a matching component cannot be found.
fn add_component(&mut self, name: &str) -> Result<Option<ComponentId>> {
fn add_component(&mut self, name: &str, features: WasmFeatures) -> Result<Option<ComponentId>> {
if let Some((id, _)) = self.graph.get_component_by_name(name) {
return Ok(Some(id));
}

match self.find_component(name)? {
match self.find_component(name, features)? {
Some(component) => Ok(Some(self.graph.add_component(component)?)),
None => Ok(None),
}
}

/// Finds the component with the given name on disk.
fn find_component(&self, name: &str) -> Result<Option<Component<'a>>> {
fn find_component(&self, name: &str, features: WasmFeatures) -> Result<Option<Component<'a>>> {
// Check the config for an explicit path (must be a valid component)
if let Some(dep) = self.config.dependencies.get(name) {
log::debug!(
Expand All @@ -127,13 +131,14 @@ impl<'a> CompositionGraphBuilder<'a> {
return Ok(Some(Component::from_file(
name,
self.config.dir.join(&dep.path),
features,
)?));
}

// Otherwise, search the paths for a valid component with the same name
log::info!("searching for a component with name `{name}`");
for dir in std::iter::once(&self.config.dir).chain(self.config.search_paths.iter()) {
if let Some(component) = Self::parse_component(dir, name)? {
if let Some(component) = Self::parse_component(dir, name, features)? {
return Ok(Some(component));
}
}
Expand All @@ -144,7 +149,11 @@ impl<'a> CompositionGraphBuilder<'a> {
/// Parses a component from the given directory, if it exists.
///
/// Returns `Ok(None)` if the component does not exist.
fn parse_component(dir: &Path, name: &str) -> Result<Option<Component<'a>>> {
fn parse_component(
dir: &Path,
name: &str,
features: WasmFeatures,
) -> Result<Option<Component<'a>>> {
let mut path = dir.join(name);

for ext in ["wasm", "wat"] {
Expand All @@ -154,7 +163,7 @@ impl<'a> CompositionGraphBuilder<'a> {
continue;
}

return Ok(Some(Component::from_file(name, &path)?));
return Ok(Some(Component::from_file(name, &path, features)?));
}

Ok(None)
Expand All @@ -165,12 +174,17 @@ impl<'a> CompositionGraphBuilder<'a> {
/// Returns an index into `instances` for the instance being instantiated.
///
/// Returns `Ok(None)` if a component to instantiate cannot be found.
fn instantiate(&mut self, name: &str, component_name: &str) -> Result<Option<(usize, bool)>> {
fn instantiate(
&mut self,
name: &str,
component_name: &str,
features: WasmFeatures,
) -> Result<Option<(usize, bool)>> {
if let Some(index) = self.instances.get_index_of(name) {
return Ok(Some((index, true)));
}

match self.add_component(component_name)? {
match self.add_component(component_name, features)? {
Some(component_id) => {
let (index, prev) = self
.instances
Expand Down Expand Up @@ -301,13 +315,18 @@ impl<'a> CompositionGraphBuilder<'a> {
/// Processes a dependency in the graph.
///
/// Returns `Ok(Some(index))` if the dependency resulted in a new dependency instance being created.
fn process_dependency(&mut self, dependency: Dependency) -> Result<Option<usize>> {
fn process_dependency(
&mut self,
dependency: Dependency,
features: WasmFeatures,
) -> Result<Option<usize>> {
match dependency.kind {
DependencyKind::Instance { instance, export } => self.process_instance_dependency(
dependency.dependent,
dependency.import,
&instance,
export.as_deref(),
features,
),
DependencyKind::Definition { index, export } => {
// The dependency is on a definition component, so we simply connect the dependent to the definition's export
Expand Down Expand Up @@ -348,6 +367,7 @@ impl<'a> CompositionGraphBuilder<'a> {
import: InstanceImportRef,
instance: &str,
export: Option<&str>,
features: WasmFeatures,
) -> Result<Option<usize>> {
let name = self.config.dependency_name(instance);

Expand All @@ -356,7 +376,7 @@ impl<'a> CompositionGraphBuilder<'a> {
dependent_name = self.instances.get_index(dependent_index).unwrap().0,
);

match self.instantiate(instance, name)? {
match self.instantiate(instance, name, features)? {
Some((instance, existing)) => {
let (dependent, import_name, import_type) = self.resolve_import_ref(import);

Expand Down Expand Up @@ -478,12 +498,12 @@ impl<'a> CompositionGraphBuilder<'a> {
}

/// Build the instantiation graph.
fn build(mut self) -> Result<(InstanceId, CompositionGraph<'a>)> {
fn build(mut self, features: WasmFeatures) -> Result<(InstanceId, CompositionGraph<'a>)> {
let mut queue: VecDeque<Dependency> = VecDeque::new();

// Instantiate the root and push its dependencies to the queue
let (root_instance, existing) = self
.instantiate(ROOT_COMPONENT_NAME, ROOT_COMPONENT_NAME)?
.instantiate(ROOT_COMPONENT_NAME, ROOT_COMPONENT_NAME, features)?
.unwrap();

assert!(!existing);
Expand All @@ -492,13 +512,11 @@ impl<'a> CompositionGraphBuilder<'a> {

// Process all remaining dependencies in the queue
while let Some(dependency) = queue.pop_front() {
if let Some(instance) = self.process_dependency(dependency)? {
if let Some(instance) = self.process_dependency(dependency, features)? {
self.push_dependencies(instance, &mut queue)?;
}
}

self.graph.unify_imported_resources();

Ok((self.instances[root_instance], self.graph))
}
}
Expand All @@ -513,6 +531,7 @@ impl<'a> CompositionGraphBuilder<'a> {
pub struct ComponentComposer<'a> {
component: &'a Path,
config: &'a Config,
features: WasmFeatures,
}

impl<'a> ComponentComposer<'a> {
Expand All @@ -521,8 +540,12 @@ impl<'a> ComponentComposer<'a> {
/// ## Arguments
/// * `component` - The path to the component to compose.
/// * `config` - The configuration to use for the composition.
pub fn new(component: &'a Path, config: &'a Config) -> Self {
Self { component, config }
pub fn new(component: &'a Path, config: &'a Config, features: WasmFeatures) -> Self {
Self {
component,
config,
features,
}
}

/// Composes a WebAssembly component based on the composer's configuration.
Expand All @@ -531,7 +554,8 @@ impl<'a> ComponentComposer<'a> {
/// Returns the bytes of the composed component.
pub fn compose(&self) -> Result<Vec<u8>> {
let (root_instance, graph) =
CompositionGraphBuilder::new(self.component, self.config)?.build()?;
CompositionGraphBuilder::new(self.component, self.config, self.features)?
.build(self.features)?;

// If only the root component was instantiated, then there are no resolved dependencies
if graph.instances.len() == 1 {
Expand Down
69 changes: 58 additions & 11 deletions crates/wasm-compose/src/encoding.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,17 @@ impl<'a> TypeEncoder<'a> {
return ret;
}

if let Some((instance, name)) = state.cur.instance_exports.get(&key) {
let ret = state.cur.encodable.type_count();
state.cur.encodable.alias(Alias::InstanceExport {
instance: *instance,
name,
kind: ComponentExportKind::Type,
});
log::trace!("id defined in current instance");
return ret;
}

match id.peel_alias(&self.0.types) {
Some(next) => id = next,
// If there's no more aliases then fall through to the
Expand All @@ -608,15 +619,17 @@ impl<'a> TypeEncoder<'a> {
return match id {
AnyTypeId::Core(ComponentCoreTypeId::Sub(_)) => unreachable!(),
AnyTypeId::Core(ComponentCoreTypeId::Module(id)) => self.module_type(state, id),
AnyTypeId::Component(id) => match id {
ComponentAnyTypeId::Resource(_) => {
unreachable!("should have been handled in `TypeEncoder::component_entity_type`")
AnyTypeId::Component(id) => {
match id {
ComponentAnyTypeId::Resource(r) => {
unreachable!("should have been handled in `TypeEncoder::component_entity_type`: {r:?}")
}
ComponentAnyTypeId::Defined(id) => self.defined_type(state, id),
ComponentAnyTypeId::Func(id) => self.component_func_type(state, id),
ComponentAnyTypeId::Instance(id) => self.component_instance_type(state, id),
ComponentAnyTypeId::Component(id) => self.component_type(state, id),
}
ComponentAnyTypeId::Defined(id) => self.defined_type(state, id),
ComponentAnyTypeId::Func(id) => self.component_func_type(state, id),
ComponentAnyTypeId::Instance(id) => self.component_instance_type(state, id),
ComponentAnyTypeId::Component(id) => self.component_type(state, id),
},
}
};
}

Expand Down Expand Up @@ -667,6 +680,9 @@ impl<'a> TypeEncoder<'a> {
state.cur.encodable.ty().defined_type().borrow(ty);
index
}
ComponentDefinedType::Future(ty) => self.future(state, *ty),
ComponentDefinedType::Stream(ty) => self.stream(state, *ty),
ComponentDefinedType::ErrorContext => self.error_context(state),
}
}

Expand Down Expand Up @@ -788,6 +804,28 @@ impl<'a> TypeEncoder<'a> {
}
export
}

fn future(&self, state: &mut TypeState<'a>, ty: Option<ct::ComponentValType>) -> u32 {
let ty = ty.map(|ty| self.component_val_type(state, ty));

let index = state.cur.encodable.type_count();
state.cur.encodable.ty().defined_type().future(ty);
index
}

fn stream(&self, state: &mut TypeState<'a>, ty: ct::ComponentValType) -> u32 {
let ty = self.component_val_type(state, ty);

let index = state.cur.encodable.type_count();
state.cur.encodable.ty().defined_type().stream(ty);
index
}

fn error_context(&self, state: &mut TypeState<'a>) -> u32 {
let index = state.cur.encodable.type_count();
state.cur.encodable.ty().defined_type().error_context();
index
}
}

/// Represents an instance index in a composition graph.
Expand Down Expand Up @@ -1215,8 +1253,11 @@ impl DependencyRegistrar<'_, '_> {
match &self.types[ty] {
ComponentDefinedType::Primitive(_)
| ComponentDefinedType::Enum(_)
| ComponentDefinedType::Flags(_) => {}
ComponentDefinedType::List(t) | ComponentDefinedType::Option(t) => self.val_type(*t),
| ComponentDefinedType::Flags(_)
| ComponentDefinedType::ErrorContext => {}
ComponentDefinedType::List(t)
| ComponentDefinedType::Option(t)
| ComponentDefinedType::Stream(t) => self.val_type(*t),
ComponentDefinedType::Own(r) | ComponentDefinedType::Borrow(r) => {
self.ty(ComponentAnyTypeId::Resource(*r))
}
Expand Down Expand Up @@ -1245,6 +1286,11 @@ impl DependencyRegistrar<'_, '_> {
self.val_type(*err);
}
}
ComponentDefinedType::Future(ty) => {
if let Some(ty) = ty {
self.val_type(*ty);
}
}
}
}
}
Expand Down Expand Up @@ -1402,7 +1448,7 @@ impl<'a> CompositionGraphEncoder<'a> {
state.push(Encodable::Instance(InstanceType::new()));
for (name, types) in exports {
let (component, ty) = types[0];
log::trace!("export {name}");
log::trace!("export {name}: {ty:?}");
let export = TypeEncoder::new(component).export(name, ty, state);
let t = match &mut state.cur.encodable {
Encodable::Instance(c) => c,
Expand All @@ -1418,6 +1464,7 @@ impl<'a> CompositionGraphEncoder<'a> {
}
}
}

let instance_type = match state.pop() {
Encodable::Instance(c) => c,
_ => unreachable!(),
Expand Down
Loading

0 comments on commit c8c3f47

Please sign in to comment.