From 06fe1160975ab5af1a1f6d8bdff796bfeefdd1e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Laferri=C3=A8re?= Date: Sat, 3 Aug 2024 17:33:43 -0400 Subject: [PATCH] Remove `Library` trait (#1422) * feat(assembler): implement serialization for `CompiledLibrary` and `KernelLibrary` * feat(assembly): Add `CompiledLibrary::write_to_dir()` and `KernelLibrary::write_to_dir()` * feat(assembler): Remove `MaslLibrary` * remove docs building * fix(assembly): ensure aliases are resolved to roots This commit fixes the issue where re-exported procedures (aliases) were not being handled properly during assembly. It now explicitly represents them in the call graph, so that topological ordering of the call graph will ensure that we only visit the aliases once the aliased procedure has been compiled/visited. As part of the solution implemented here, some refinements to `AliasTarget` were made, in order to explicitly represent whether the target is absolute or not (just like `InvocationTarget`). Additionally, to avoid confusion, `FullyQualifiedProcedureName` was renamed to `QualifiedProcedureName`, so as to make it clear that just because the path is qualified, does not mean it is absolute (and thus "fully" qualified). Some conveniences were added to `LibraryNamespace`, `LibraryPath`, and `AliasTarget` to make certain operations/conversions more ergonomic and expressive. * feat: add pretty-print helper for lists of values * feat: support assembling with compiled libraries This commit refactors `CompiledLibrary` a bit, to remove some unnecessary restrictions leftover from the old MASL libraries: * A `CompiledLibrary` no longer has a name, but it has a content digest obtained by lexicographically ordering the exported MAST roots of the library, and merging the hashes in order. * As a consequence of being unnamed/having no namespace, a `CompiledLibrary` can now consist of procedures from many modules _and_ many namespaces. Any limitation we impose on top of that can be done via wrapper types, like how `KernelLibrary` is implemented. * Support for re-exported procedures in a `CompiledLibrary` is implemented. It is assumed that all required libraries will be provided to the `Host` when executing a program. * Some ergonomic improvements to APIs which accept libraries or sets of modules, to allow a greater variety of ways you can pass them. * fix(assembly): address conflicting procedure definitions bug Previously, we assumed that two procedures with the same MAST root, but differing numbers of locals, was a bug in the assembler. However, this is not the case, as I will elaborate on below. If you compile a program like so: ```masm use.std::u64 begin exec.u64::checked_and end ``` The resulting MAST would look something like: ```mast begin external.0x.... end ``` This MAST will have the exact same MAST root as `std::u64::checked_and`, because `external` nodes have the same digest as the node they refer to. Now, if the exec'd procedure has the same number of locals as the caller, this is presumed to be a "compatible" procedure, meaning it is fine to let both procedures refer to the same MAST. However, when the number of procedure locals _differs_, we were raising a compatible definition error, because it was assumed that due to the instructions added when procedure locals are present, two procedures with differing numbers of locals _could not_ have the same root by definition. This is incorrect, let me illustrate: ```masm export.foo.2 ... end use.foo proc.bar.3 exec.foo::foo end begin exec.foo::foo end ``` Assume that `foo.masm` is first compiled to a `CompiledLibrary`, which is then added to the assembler when assembling an executable program from `bar.masm`. Also, recall that `begin .. end` blocks are effectively an exported procedure definition, with zero locals, that has a special name - but in all other ways it is just a normal procedure. The above program is perfectly legal, but we would raise an error during assembly of the program due to the `begin .. end` block compiling to an `external` node referencing the MAST root of `foo::foo`, which has a non-zero number of procedure locals. The `bar` procedure is there to illustrate that even though it too simply "wraps" the `foo::foo` procedure, it has a non-zero number of procedure locals, and thus cannot ever have the same MAST root as a wrapped procedure with a non-zero number of locals, due to the presence of locals changing the root of the wrapping procedure. A check has been kept around that ensures we catch if ever there are two procedures with non-zero counts of procedure locals, with the same MAST root. * chore: update changelog --------- Co-authored-by: Bobbin Threadbare Co-authored-by: Paul Schoenfelder --- CHANGELOG.md | 1 + Cargo.lock | 16 +- .../src/assembler/instruction/procedures.rs | 11 +- assembly/src/assembler/mast_forest_builder.rs | 53 +- assembly/src/assembler/mod.rs | 212 ++++--- assembly/src/assembler/module_graph/mod.rs | 71 +-- .../assembler/module_graph/name_resolver.rs | 15 +- .../assembler/module_graph/rewrites/module.rs | 12 +- assembly/src/assembler/procedure.rs | 14 +- assembly/src/assembler/tests.rs | 156 +++--- assembly/src/ast/invocation_target.rs | 1 + assembly/src/ast/module.rs | 23 +- assembly/src/ast/procedure/alias.rs | 146 +++-- assembly/src/ast/procedure/mod.rs | 2 +- assembly/src/ast/procedure/name.rs | 52 +- assembly/src/ast/procedure/resolver.rs | 4 +- assembly/src/ast/visit.rs | 38 +- assembly/src/compile.rs | 2 +- assembly/src/errors.rs | 19 +- assembly/src/lib.rs | 4 +- assembly/src/library/error.rs | 22 +- assembly/src/library/masl.rs | 337 +----------- assembly/src/library/mod.rs | 520 +++++++++++++++--- assembly/src/library/namespace.rs | 12 + assembly/src/library/path.rs | 38 +- assembly/src/library/tests.rs | 42 +- assembly/src/parser/grammar.lalrpop | 10 +- assembly/src/sema/mod.rs | 15 +- assembly/src/testing.rs | 10 +- assembly/src/tests.rs | 160 ++---- core/src/lib.rs | 15 + core/src/mast/mod.rs | 5 + miden/benches/program_compilation.rs | 6 +- miden/benches/program_execution.rs | 5 +- miden/src/cli/bundle.rs | 16 +- miden/src/cli/data.rs | 26 +- miden/src/examples/blake3.rs | 2 +- miden/src/repl/mod.rs | 20 +- miden/src/tools/mod.rs | 2 +- miden/tests/integration/cli/cli_test.rs | 5 +- miden/tests/integration/flow_control/mod.rs | 40 +- .../integration/operations/io_ops/env_ops.rs | 4 +- stdlib/build.rs | 68 +-- stdlib/md_renderer.rs | 117 ---- stdlib/src/lib.rs | 60 +- stdlib/tests/crypto/falcon.rs | 2 +- stdlib/tests/mem/mod.rs | 8 +- test-utils/src/lib.rs | 95 +++- test-utils/src/test_builders.rs | 8 +- 49 files changed, 1243 insertions(+), 1279 deletions(-) delete mode 100644 stdlib/md_renderer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e5e5b77317..967917ab3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ - Added functions to `MastForest` and `MastForestBuilder` to add and ensure nodes with fewer LOC (#1404, #1412) - Added `Assembler::assemble_library()` (#1413). - Added `Assembler::assemble_kernel()` (#1418). +- Added `miden_core::prettier::pretty_print_csv` helper, for formatting of iterators over `PrettyPrint` values as comma-separated items #### Changed diff --git a/Cargo.lock b/Cargo.lock index 5780b29cb3..6dc689e1b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -967,7 +967,7 @@ dependencies = [ [[package]] name = "miden-air" -version = "0.9.2" +version = "0.10.0" dependencies = [ "criterion", "miden-core", @@ -979,7 +979,7 @@ dependencies = [ [[package]] name = "miden-assembly" -version = "0.9.2" +version = "0.10.0" dependencies = [ "aho-corasick", "lalrpop", @@ -997,7 +997,7 @@ dependencies = [ [[package]] name = "miden-core" -version = "0.9.1" +version = "0.10.0" dependencies = [ "miden-crypto", "miden-formatting", @@ -1050,7 +1050,7 @@ dependencies = [ [[package]] name = "miden-processor" -version = "0.9.2" +version = "0.10.0" dependencies = [ "logtest", "miden-air", @@ -1065,7 +1065,7 @@ dependencies = [ [[package]] name = "miden-prover" -version = "0.9.1" +version = "0.10.0" dependencies = [ "elsa", "miden-air", @@ -1078,7 +1078,7 @@ dependencies = [ [[package]] name = "miden-stdlib" -version = "0.9.2" +version = "0.10.0" dependencies = [ "blake3", "miden-air", @@ -1114,7 +1114,7 @@ dependencies = [ [[package]] name = "miden-verifier" -version = "0.9.1" +version = "0.10.0" dependencies = [ "miden-air", "miden-core", @@ -1124,7 +1124,7 @@ dependencies = [ [[package]] name = "miden-vm" -version = "0.9.1" +version = "0.10.0" dependencies = [ "assert_cmd", "blake3", diff --git a/assembly/src/assembler/instruction/procedures.rs b/assembly/src/assembler/instruction/procedures.rs index 4863fb49f7..3aad75c5f3 100644 --- a/assembly/src/assembler/instruction/procedures.rs +++ b/assembly/src/assembler/instruction/procedures.rs @@ -74,19 +74,12 @@ impl Assembler { proc_ctx.register_external_call(&proc, false)?; } Some(proc) => proc_ctx.register_external_call(&proc, false)?, - None if matches!(kind, InvokeKind::SysCall) => { - return Err(AssemblyError::UnknownSysCallTarget { - span, - source_file: current_source_file.clone(), - callee: mast_root, - }); - } None => (), } let mast_root_node_id = { match kind { - InvokeKind::Exec => { + InvokeKind::Exec | InvokeKind::ProcRef => { // Note that here we rely on the fact that we topologically sorted the // procedures, such that when we assemble a procedure, all // procedures that it calls will have been assembled, and @@ -161,7 +154,7 @@ impl Assembler { mast_forest_builder: &MastForestBuilder, ) -> Result<(), AssemblyError> { let digest = - self.resolve_target(InvokeKind::Exec, callee, proc_ctx, mast_forest_builder)?; + self.resolve_target(InvokeKind::ProcRef, callee, proc_ctx, mast_forest_builder)?; self.procref_mast_root(digest, proc_ctx, span_builder, mast_forest_builder) } diff --git a/assembly/src/assembler/mast_forest_builder.rs b/assembly/src/assembler/mast_forest_builder.rs index 7bd7863adf..23336b6a23 100644 --- a/assembly/src/assembler/mast_forest_builder.rs +++ b/assembly/src/assembler/mast_forest_builder.rs @@ -20,6 +20,7 @@ pub struct MastForestBuilder { mast_forest: MastForest, node_id_by_hash: BTreeMap, procedures: BTreeMap>, + procedure_hashes: BTreeMap, proc_gid_by_hash: BTreeMap, } @@ -39,6 +40,13 @@ impl MastForestBuilder { self.procedures.get(&gid).cloned() } + /// Returns the hash of the procedure with the specified [`GlobalProcedureIndex`], or None if + /// such a procedure is not present in this MAST forest builder. + #[inline(always)] + pub fn get_procedure_hash(&self, gid: GlobalProcedureIndex) -> Option { + self.procedure_hashes.get(&gid).cloned() + } + /// Returns a reference to the procedure with the specified MAST root, or None /// if such a procedure is not present in this MAST forest builder. #[inline(always)] @@ -61,6 +69,17 @@ impl MastForestBuilder { } impl MastForestBuilder { + pub fn insert_procedure_hash( + &mut self, + gid: GlobalProcedureIndex, + proc_hash: RpoDigest, + ) -> Result<(), AssemblyError> { + // TODO(plafer): Check if exists + self.procedure_hashes.insert(gid, proc_hash); + + Ok(()) + } + /// Inserts a procedure into this MAST forest builder. /// /// If the procedure with the same ID already exists in this forest builder, this will have @@ -76,17 +95,9 @@ impl MastForestBuilder { // // If there is already a cache entry, but it conflicts with what we're trying to cache, // then raise an error. - if let Some(cached) = self.procedures.get(&gid) { - let cached_root = self.mast_forest[cached.body_node_id()].digest(); - if cached_root != proc_root || cached.num_locals() != procedure.num_locals() { - return Err(AssemblyError::ConflictingDefinitions { - first: cached.fully_qualified_name().clone(), - second: procedure.fully_qualified_name().clone(), - }); - } - + if self.procedures.contains_key(&gid) { // The global procedure index and the MAST root resolve to an already cached version of - // this procedure, nothing to do. + // this procedure, or an alias of it, nothing to do. // // TODO: We should emit a warning for this, because while it is not an error per se, it // does reflect that we're doing work we don't need to be doing. However, emitting a @@ -98,23 +109,29 @@ impl MastForestBuilder { // We don't have a cache entry yet, but we do want to make sure we don't have a conflicting // cache entry with the same MAST root: if let Some(cached) = self.find_procedure(&proc_root) { - if cached.num_locals() != procedure.num_locals() { + // Handle the case where a procedure with no locals is lowered to a MastForest + // consisting only of an `External` node to another procedure which has one or more + // locals. This will result in the calling procedure having the same digest as the + // callee, but the two procedures having mismatched local counts. When this occurs, + // we want to use the procedure with non-zero local count as the definition, and treat + // the other procedure as an alias, which can be referenced like any other procedure, + // but the MAST returned for it will be that of the "real" definition. + let cached_locals = cached.num_locals(); + let procedure_locals = procedure.num_locals(); + let mismatched_locals = cached_locals != procedure_locals; + let is_valid = + !mismatched_locals || core::cmp::min(cached_locals, procedure_locals) == 0; + if !is_valid { return Err(AssemblyError::ConflictingDefinitions { first: cached.fully_qualified_name().clone(), second: procedure.fully_qualified_name().clone(), }); } - - // We have a previously cached version of an equivalent procedure, just under a - // different [GlobalProcedureIndex], so insert the cached procedure into the slot for - // `id`, but skip inserting a record in the MAST root lookup table - self.make_root(procedure.body_node_id()); - self.procedures.insert(gid, Arc::new(procedure)); - return Ok(()); } self.make_root(procedure.body_node_id()); self.proc_gid_by_hash.insert(proc_root, gid); + self.insert_procedure_hash(gid, procedure.mast_root())?; self.procedures.insert(gid, Arc::new(procedure)); Ok(()) diff --git a/assembly/src/assembler/mod.rs b/assembly/src/assembler/mod.rs index ba0bd395d4..a262434f86 100644 --- a/assembly/src/assembler/mod.rs +++ b/assembly/src/assembler/mod.rs @@ -1,12 +1,11 @@ use crate::{ - ast::{self, FullyQualifiedProcedureName, InvocationTarget, InvokeKind, ModuleKind}, + ast::{self, Export, InvocationTarget, InvokeKind, ModuleKind, QualifiedProcedureName}, diagnostics::Report, library::{CompiledLibrary, KernelLibrary}, sema::SemanticAnalysisError, - AssemblyError, Compile, CompileOptions, Library, LibraryNamespace, LibraryPath, RpoDigest, - Spanned, + AssemblyError, Compile, CompileOptions, LibraryNamespace, LibraryPath, RpoDigest, Spanned, }; -use alloc::{sync::Arc, vec::Vec}; +use alloc::{collections::BTreeMap, vec::Vec}; use mast_forest_builder::MastForestBuilder; use module_graph::{ProcedureWrapper, WrappedModule}; use vm_core::{mast::MastNodeId, Decorator, DecoratorList, Felt, Kernel, Operation, Program}; @@ -45,8 +44,8 @@ use self::module_graph::{CallerInfo, ModuleGraph, ResolvedTarget}; /// [Assembler::assemble_program]. /// * If you want to link your executable to a few other modules that implement supporting /// procedures, build the assembler with them first, using the various builder methods on -/// [Assembler], e.g. [Assembler::with_module], [Assembler::with_library], etc. Then, call -/// [Assembler::assemble_program] to get your compiled program. +/// [Assembler], e.g. [Assembler::with_module], [Assembler::with_compiled_library], etc. Then, +/// call [Assembler::assemble_program] to get your compiled program. #[derive(Clone, Default)] pub struct Assembler { /// The global [ModuleGraph] for this assembler. @@ -139,64 +138,24 @@ impl Assembler { } /// Adds the compiled library to provide modules for the compilation. - pub fn add_compiled_library(&mut self, library: CompiledLibrary) -> Result<(), Report> { + pub fn add_compiled_library( + &mut self, + library: impl AsRef, + ) -> Result<(), Report> { self.module_graph - .add_compiled_modules(library.into_module_infos()) + .add_compiled_modules(library.as_ref().module_infos()) .map_err(Report::from)?; Ok(()) } - /// Adds the library to provide modules for the compilation. - pub fn with_library(mut self, library: &L) -> Result - where - L: ?Sized + Library + 'static, - { - self.add_library(library)?; - - Ok(self) - } - - /// Adds the library to provide modules for the compilation. - pub fn add_library(&mut self, library: &L) -> Result<(), Report> - where - L: ?Sized + Library + 'static, - { - let namespace = library.root_ns(); - library.modules().try_for_each(|module| { - if !module.is_in_namespace(namespace) { - return Err(Report::new(AssemblyError::InconsistentNamespace { - expected: namespace.clone(), - actual: module.namespace().clone(), - })); - } - - self.add_module(module)?; - - Ok(()) - }) - } - - /// Adds a library bundle to provide modules for the compilation. - pub fn with_libraries<'a, I, L>(mut self, libraries: I) -> Result - where - L: ?Sized + Library + 'static, - I: IntoIterator, - { - self.add_libraries(libraries)?; + /// Adds the compiled library to provide modules for the compilation. + pub fn with_compiled_library( + mut self, + library: impl AsRef, + ) -> Result { + self.add_compiled_library(library)?; Ok(self) } - - /// Adds a library bundle to provide modules for the compilation. - pub fn add_libraries<'a, I, L>(&mut self, libraries: I) -> Result<(), Report> - where - L: ?Sized + Library + 'static, - I: IntoIterator, - { - for library in libraries { - self.add_library(library)?; - } - Ok(()) - } } // ------------------------------------------------------------------------------------------------ @@ -236,30 +195,41 @@ impl Assembler { /// Returns an error if parsing or compilation of the specified modules fails. pub fn assemble_library( mut self, - modules: impl Iterator, + modules: impl IntoIterator, ) -> Result { - let ast_module_indices: Vec = modules - .map(|module| { - let module = module.compile_with_options(CompileOptions::for_library())?; + let ast_module_indices = + modules.into_iter().try_fold(Vec::default(), |mut acc, module| { + module + .compile_with_options(CompileOptions::for_library()) + .and_then(|module| { + self.module_graph.add_ast_module(module).map_err(Report::from) + }) + .map(move |module_id| { + acc.push(module_id); + acc + }) + })?; - Ok(self.module_graph.add_ast_module(module)?) - }) - .collect::>()?; self.module_graph.recompute()?; let mut mast_forest_builder = MastForestBuilder::default(); let exports = { - let mut exports = Vec::new(); + let mut exports = BTreeMap::new(); for module_idx in ast_module_indices { - // Note: it is safe to use `unwrap_ast()` here, since all modules looped over are - // AST (we just added them to the module graph) + // Note: it is safe to use `unwrap_ast()` here, since all of the modules contained + // in `ast_module_indices` are in AST form by definition. let ast_module = self.module_graph[module_idx].unwrap_ast().clone(); for (proc_idx, fqn) in ast_module.exported_procedures() { - self.compile_subgraph(module_idx + proc_idx, false, &mut mast_forest_builder)?; - exports.push(fqn); + let gid = module_idx + proc_idx; + self.compile_subgraph(gid, &mut mast_forest_builder)?; + + let proc_hash = mast_forest_builder + .get_procedure_hash(gid) + .expect("compilation succeeded but root not found in cache"); + exports.insert(fqn, proc_hash); } } @@ -295,10 +265,15 @@ impl Assembler { let exports = ast_module .exported_procedures() .map(|(proc_idx, fqn)| { - self.compile_subgraph(module_idx + proc_idx, false, &mut mast_forest_builder)?; - Ok(fqn) + let gid = module_idx + proc_idx; + self.compile_subgraph(gid, &mut mast_forest_builder)?; + + let proc_hash = mast_forest_builder + .get_procedure_hash(gid) + .expect("compilation succeeded but root not found in cache"); + Ok((fqn, proc_hash)) }) - .collect::, Report>>()?; + .collect::, Report>>()?; let library = CompiledLibrary::new(mast_forest_builder.build(), exports)?; Ok(library.try_into()?) @@ -338,7 +313,10 @@ impl Assembler { // Compile the module graph rooted at the entrypoint let mut mast_forest_builder = MastForestBuilder::default(); - let entry_procedure = self.compile_subgraph(entrypoint, true, &mut mast_forest_builder)?; + self.compile_subgraph(entrypoint, &mut mast_forest_builder)?; + let entry_procedure = mast_forest_builder + .get_procedure(entrypoint) + .expect("compilation succeeded but root not found in cache"); Ok(Program::with_kernel( mast_forest_builder.build(), @@ -354,9 +332,8 @@ impl Assembler { fn compile_subgraph( &mut self, root: GlobalProcedureIndex, - is_entrypoint: bool, mast_forest_builder: &mut MastForestBuilder, - ) -> Result, Report> { + ) -> Result<(), Report> { let mut worklist: Vec = self .module_graph .topological_sort_from_root(root) @@ -376,34 +353,23 @@ impl Assembler { assert!(!worklist.is_empty()); - let compiled = if is_entrypoint { - self.process_graph_worklist(&mut worklist, Some(root), mast_forest_builder)? - } else { - let _ = self.process_graph_worklist(&mut worklist, None, mast_forest_builder)?; - mast_forest_builder.get_procedure(root) - }; - - Ok(compiled.expect("compilation succeeded but root not found in cache")) + self.process_graph_worklist(&mut worklist, mast_forest_builder) } /// Compiles all procedures in the `worklist`. fn process_graph_worklist( &mut self, worklist: &mut Vec, - entrypoint: Option, mast_forest_builder: &mut MastForestBuilder, - ) -> Result>, Report> { + ) -> Result<(), Report> { // Process the topological ordering in reverse order (bottom-up), so that // each procedure is compiled with all of its dependencies fully compiled - let mut compiled_entrypoint = None; while let Some(procedure_gid) = worklist.pop() { // If we have already compiled this procedure, do not recompile if let Some(proc) = mast_forest_builder.get_procedure(procedure_gid) { self.module_graph.register_mast_root(procedure_gid, proc.mast_root())?; continue; } - let is_entry = entrypoint == Some(procedure_gid); - // Fetch procedure metadata from the graph let module = match &self.module_graph[procedure_gid.module] { WrappedModule::Ast(ast_module) => ast_module, @@ -411,33 +377,51 @@ impl Assembler { // compile. WrappedModule::Info(_) => continue, }; - let ast = &module[procedure_gid.index]; - let num_locals = ast.num_locals(); - let name = FullyQualifiedProcedureName { - span: ast.span(), - module: module.path().clone(), - name: ast.name().clone(), - }; - let pctx = ProcedureContext::new(procedure_gid, name, ast.visibility()) - .with_num_locals(num_locals as u16) - .with_span(ast.span()) - .with_source_file(ast.source_file()); - - // Compile this procedure - let procedure = self.compile_procedure(pctx, mast_forest_builder)?; - - // Cache the compiled procedure, unless it's the program entrypoint - if is_entry { - mast_forest_builder.make_root(procedure.body_node_id()); - compiled_entrypoint = Some(Arc::from(procedure)); - } else { - // Make the MAST root available to all dependents - self.module_graph.register_mast_root(procedure_gid, procedure.mast_root())?; - mast_forest_builder.insert_procedure(procedure_gid, procedure)?; + let export = &module[procedure_gid.index]; + match export { + Export::Procedure(proc) => { + let num_locals = proc.num_locals(); + let name = QualifiedProcedureName { + span: proc.span(), + module: module.path().clone(), + name: proc.name().clone(), + }; + let pctx = ProcedureContext::new(procedure_gid, name, proc.visibility()) + .with_num_locals(num_locals) + .with_span(proc.span()) + .with_source_file(proc.source_file()); + + // Compile this procedure + let procedure = self.compile_procedure(pctx, mast_forest_builder)?; + + // Cache the compiled procedure. + self.module_graph.register_mast_root(procedure_gid, procedure.mast_root())?; + mast_forest_builder.insert_procedure(procedure_gid, procedure)?; + } + Export::Alias(proc_alias) => { + let name = QualifiedProcedureName { + span: proc_alias.span(), + module: module.path().clone(), + name: proc_alias.name().clone(), + }; + let pctx = ProcedureContext::new(procedure_gid, name, ast::Visibility::Public) + .with_span(proc_alias.span()) + .with_source_file(proc_alias.source_file()); + + let proc_alias_root = self.resolve_target( + InvokeKind::ProcRef, + &proc_alias.target().into(), + &pctx, + mast_forest_builder, + )?; + // Make the MAST root available to all dependents + self.module_graph.register_mast_root(procedure_gid, proc_alias_root)?; + mast_forest_builder.insert_procedure_hash(procedure_gid, proc_alias_root)?; + } } } - Ok(compiled_entrypoint) + Ok(()) } /// Compiles a single Miden Assembly procedure to its MAST representation. @@ -586,8 +570,8 @@ impl Assembler { match resolved { ResolvedTarget::Phantom(digest) => Ok(digest), ResolvedTarget::Exact { gid } | ResolvedTarget::Resolved { gid, .. } => { - match mast_forest_builder.get_procedure(gid) { - Some(p) => Ok(p.mast_root()), + match mast_forest_builder.get_procedure_hash(gid) { + Some(proc_hash) => Ok(proc_hash), None => match self.module_graph.get_procedure_unsafe(gid) { ProcedureWrapper::Info(p) => Ok(p.digest), ProcedureWrapper::Ast(_) => panic!("Did not find procedure {gid:?} neither in module graph nor procedure cache"), diff --git a/assembly/src/assembler/module_graph/mod.rs b/assembly/src/assembler/module_graph/mod.rs index e0835f96f3..fb5424645e 100644 --- a/assembly/src/assembler/module_graph/mod.rs +++ b/assembly/src/assembler/module_graph/mod.rs @@ -15,11 +15,9 @@ use smallvec::{smallvec, SmallVec}; use self::{analysis::MaybeRewriteCheck, name_resolver::NameResolver, rewrites::ModuleRewriter}; use super::{GlobalProcedureIndex, ModuleIndex}; +use crate::ast::InvokeKind; use crate::{ - ast::{ - Export, FullyQualifiedProcedureName, InvocationTarget, Module, ProcedureIndex, - ProcedureName, ResolvedProcedure, - }, + ast::{Export, InvocationTarget, Module, ProcedureIndex, ProcedureName, ResolvedProcedure}, library::{ModuleInfo, ProcedureInfo}, AssemblyError, LibraryNamespace, LibraryPath, RpoDigest, Spanned, }; @@ -170,9 +168,10 @@ impl ModuleGraph { /// Adds all module infos to the graph. pub fn add_compiled_modules( &mut self, - module_infos: impl Iterator, + module_infos: impl IntoIterator, ) -> Result, AssemblyError> { let module_indices: Vec = module_infos + .into_iter() .map(|module| self.add_module(PendingWrappedModule::Info(module))) .collect::>()?; @@ -254,7 +253,7 @@ impl ModuleGraph { // TODO: simplify this to avoid using Self::add_compiled_modules() let mut graph = Self::default(); let module_indexes = graph - .add_compiled_modules([kernel_module].into_iter()) + .add_compiled_modules([kernel_module]) .expect("failed to add kernel module to the module graph"); assert_eq!(module_indexes[0], ModuleIndex::new(0), "kernel should be the first module"); @@ -337,7 +336,7 @@ impl ModuleGraph { // Apply module to call graph match pending_module { PendingWrappedModule::Ast(pending_module) => { - for (index, procedure) in pending_module.procedures().enumerate() { + for (index, _) in pending_module.procedures().enumerate() { let procedure_id = ProcedureIndex::new(index); let global_id = GlobalProcedureIndex { module: module_id, @@ -347,9 +346,7 @@ impl ModuleGraph { // Ensure all entrypoints and exported symbols are represented in the call // graph, even if they have no edges, we need them // in the graph for the topological sort - if matches!(procedure, Export::Procedure(_)) { - self.callgraph.get_or_insert_node(global_id); - } + self.callgraph.get_or_insert_node(global_id); } } PendingWrappedModule::Info(pending_module) => { @@ -391,6 +388,23 @@ impl ModuleGraph { index: procedure_id, }; + // Add edge to the call graph to represent dependency on aliased procedures + if let Export::Alias(ref alias) = procedure { + let caller = CallerInfo { + span: alias.span(), + source_file: ast_module.source_file(), + module: module_id, + kind: InvokeKind::ProcRef, + }; + let target = alias.target().into(); + if let Some(callee) = + resolver.resolve_target(&caller, &target)?.into_global_id() + { + edges.push((gid, callee)); + } + } + + // Add edges to all transitive dependencies of this procedure due to calls for invoke in procedure.invoked() { let caller = CallerInfo { span: invoke.span(), @@ -535,43 +549,6 @@ impl ModuleGraph { Entry::Occupied(ref mut entry) => { let prev_id = entry.get()[0]; if prev_id != id { - let prev_proc = { - match &self.modules[prev_id.module.as_usize()] { - WrappedModule::Ast(module) => Some(&module[prev_id.index]), - WrappedModule::Info(_) => None, - } - }; - let current_proc = { - match &self.modules[id.module.as_usize()] { - WrappedModule::Ast(module) => Some(&module[id.index]), - WrappedModule::Info(_) => None, - } - }; - - // Note: For compiled procedures, we can't check further if they're compatible, - // so we assume they are. - if let (Some(prev_proc), Some(current_proc)) = (prev_proc, current_proc) { - if prev_proc.num_locals() != current_proc.num_locals() { - // Multiple procedures with the same root, but incompatible - let prev_module = self.modules[prev_id.module.as_usize()].path(); - let prev_name = FullyQualifiedProcedureName { - span: prev_proc.span(), - module: prev_module.clone(), - name: prev_proc.name().clone(), - }; - let current_module = self.modules[id.module.as_usize()].path(); - let current_name = FullyQualifiedProcedureName { - span: current_proc.span(), - module: current_module.clone(), - name: current_proc.name().clone(), - }; - return Err(AssemblyError::ConflictingDefinitions { - first: prev_name, - second: current_name, - }); - } - } - // Multiple procedures with the same root, but compatible entry.get_mut().push(id); } diff --git a/assembly/src/assembler/module_graph/name_resolver.rs b/assembly/src/assembler/module_graph/name_resolver.rs index 2db16b000c..93d325fc72 100644 --- a/assembly/src/assembler/module_graph/name_resolver.rs +++ b/assembly/src/assembler/module_graph/name_resolver.rs @@ -4,7 +4,7 @@ use super::{ModuleGraph, WrappedModule}; use crate::{ assembler::{GlobalProcedureIndex, ModuleIndex}, ast::{ - FullyQualifiedProcedureName, Ident, InvocationTarget, InvokeKind, Module, ProcedureName, + Ident, InvocationTarget, InvokeKind, Module, ProcedureName, QualifiedProcedureName, ResolvedProcedure, }, diagnostics::{RelatedLabel, SourceFile}, @@ -140,7 +140,7 @@ impl<'a> NameResolver<'a> { module: ref imported_module, } => match self.resolve_import(caller, imported_module) { Some(imported_module) => { - let fqn = FullyQualifiedProcedureName { + let fqn = QualifiedProcedureName { span: target.span(), module: imported_module.into_inner().clone(), name: name.clone(), @@ -171,7 +171,7 @@ impl<'a> NameResolver<'a> { }), }, InvocationTarget::AbsoluteProcedurePath { ref name, ref path } => { - let fqn = FullyQualifiedProcedureName { + let fqn = QualifiedProcedureName { span: target.span(), module: path.clone(), name: name.clone(), @@ -288,13 +288,14 @@ impl<'a> NameResolver<'a> { fn find( &self, caller: &CallerInfo, - callee: &FullyQualifiedProcedureName, + callee: &QualifiedProcedureName, ) -> Result { - // If the caller is a syscall, set the invoke kind to exec until we have resolved the - // procedure, then verify that it is in the kernel module + // If the caller is a syscall, set the invoke kind to `ProcRef` until we have resolved the + // procedure, then verify that it is in the kernel module. This bypasses validation until + // after resolution let mut current_caller = if matches!(caller.kind, InvokeKind::SysCall) { let mut caller = caller.clone(); - caller.kind = InvokeKind::Exec; + caller.kind = InvokeKind::ProcRef; Cow::Owned(caller) } else { Cow::Borrowed(caller) diff --git a/assembly/src/assembler/module_graph/rewrites/module.rs b/assembly/src/assembler/module_graph/rewrites/module.rs index 4637ff2f4e..7ce8554e38 100644 --- a/assembly/src/assembler/module_graph/rewrites/module.rs +++ b/assembly/src/assembler/module_graph/rewrites/module.rs @@ -8,7 +8,7 @@ use crate::{ }, ast::{ visit::{self, VisitMut}, - InvocationTarget, Invoke, InvokeKind, Module, Procedure, + AliasTarget, InvocationTarget, Invoke, InvokeKind, Module, Procedure, }, diagnostics::SourceFile, AssemblyError, Spanned, @@ -112,4 +112,14 @@ impl<'a, 'b: 'a> VisitMut for ModuleRewriter<'a, 'b> { ) -> ControlFlow { self.rewrite_target(InvokeKind::Exec, target) } + fn visit_mut_alias_target(&mut self, target: &mut AliasTarget) -> ControlFlow { + if matches!(target, AliasTarget::MastRoot(_)) { + return ControlFlow::Continue(()); + } + let mut invoke_target = (target as &AliasTarget).into(); + self.rewrite_target(InvokeKind::ProcRef, &mut invoke_target)?; + // This will always succeed, as the original target is qualified by construction + *target = AliasTarget::try_from(invoke_target).unwrap(); + ControlFlow::Continue(()) + } } diff --git a/assembly/src/assembler/procedure.rs b/assembly/src/assembler/procedure.rs index 18ece32c0e..b19b05c9fb 100644 --- a/assembly/src/assembler/procedure.rs +++ b/assembly/src/assembler/procedure.rs @@ -2,7 +2,7 @@ use alloc::{collections::BTreeSet, sync::Arc}; use super::GlobalProcedureIndex; use crate::{ - ast::{FullyQualifiedProcedureName, ProcedureName, Visibility}, + ast::{ProcedureName, QualifiedProcedureName, Visibility}, diagnostics::SourceFile, AssemblyError, LibraryPath, RpoDigest, SourceSpan, Spanned, }; @@ -18,7 +18,7 @@ pub struct ProcedureContext { gid: GlobalProcedureIndex, span: SourceSpan, source_file: Option>, - name: FullyQualifiedProcedureName, + name: QualifiedProcedureName, visibility: Visibility, num_locals: u16, callset: CallSet, @@ -29,7 +29,7 @@ pub struct ProcedureContext { impl ProcedureContext { pub fn new( gid: GlobalProcedureIndex, - name: FullyQualifiedProcedureName, + name: QualifiedProcedureName, visibility: Visibility, ) -> Self { Self { @@ -66,7 +66,7 @@ impl ProcedureContext { self.gid } - pub fn name(&self) -> &FullyQualifiedProcedureName { + pub fn name(&self) -> &QualifiedProcedureName { &self.name } @@ -163,7 +163,7 @@ impl Spanned for ProcedureContext { pub struct Procedure { span: SourceSpan, source_file: Option>, - path: FullyQualifiedProcedureName, + path: QualifiedProcedureName, visibility: Visibility, num_locals: u32, /// The MAST root of the procedure. @@ -178,7 +178,7 @@ pub struct Procedure { /// Constructors impl Procedure { fn new( - path: FullyQualifiedProcedureName, + path: QualifiedProcedureName, visibility: Visibility, num_locals: u32, mast_root: RpoDigest, @@ -222,7 +222,7 @@ impl Procedure { } /// Returns a reference to the fully-qualified name of this procedure - pub fn fully_qualified_name(&self) -> &FullyQualifiedProcedureName { + pub fn fully_qualified_name(&self) -> &QualifiedProcedureName { &self.path } diff --git a/assembly/src/assembler/tests.rs b/assembly/src/assembler/tests.rs index cfec29207d..ebde3e94ba 100644 --- a/assembly/src/assembler/tests.rs +++ b/assembly/src/assembler/tests.rs @@ -1,72 +1,43 @@ -use alloc::{boxed::Box, vec::Vec}; +use core::iter; + use pretty_assertions::assert_eq; use vm_core::{assert_matches, mast::MastForest, Program}; -use super::{Assembler, Library, Operation}; +use super::{Assembler, Operation}; use crate::{ assembler::{combine_mast_node_ids, mast_forest_builder::MastForestBuilder}, ast::{Module, ModuleKind}, - LibraryNamespace, Version, }; // TESTS // ================================================================================================ -// TODO: Fix test after we implement the new `Assembler::add_library()` -#[ignore] -#[allow(unused)] #[test] fn nested_blocks() { - const MODULE: &str = "foo::bar"; const KERNEL: &str = r#" export.foo add end"#; - const PROCEDURE: &str = r#" + const MODULE: &str = "foo::bar"; + const MODULE_PROCEDURE: &str = r#" export.baz push.29 end"#; - pub struct DummyLibrary { - namespace: LibraryNamespace, - #[allow(clippy::vec_box)] - modules: Vec>, - dependencies: Vec, - } - - impl Default for DummyLibrary { - fn default() -> Self { - let ast = - Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, PROCEDURE).unwrap(); - let namespace = ast.namespace().clone(); - Self { - namespace, - modules: vec![ast], - dependencies: Vec::new(), - } - } - } - - impl Library for DummyLibrary { - fn root_ns(&self) -> &LibraryNamespace { - &self.namespace - } - - fn version(&self) -> &Version { - const MIN: Version = Version::min(); - &MIN - } - - fn modules(&self) -> impl ExactSizeIterator + '_ { - self.modules.iter().map(|m| m.as_ref()) - } - - fn dependencies(&self) -> &[LibraryNamespace] { - &self.dependencies - } - } - - let assembler = Assembler::default().with_library(&DummyLibrary::default()).unwrap(); + let assembler = { + let kernel_lib = Assembler::default().assemble_kernel(KERNEL).unwrap(); + + let dummy_module = + Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, MODULE_PROCEDURE) + .unwrap(); + let dummy_library = + Assembler::default().assemble_library(iter::once(dummy_module)).unwrap(); + + let mut assembler = Assembler::with_kernel(kernel_lib); + assembler.add_compiled_library(dummy_library).unwrap(); + + assembler + }; // The expected `MastForest` for the program (that we will build by hand) let mut expected_mast_forest_builder = MastForestBuilder::default(); @@ -192,8 +163,9 @@ fn nested_blocks() { let expected_program = Program::new(expected_mast_forest_builder.build(), combined_node_id); assert_eq!(expected_program.hash(), program.hash()); - // also check that the program has the right number of procedures - assert_eq!(program.num_procedures(), 5); + // also check that the program has the right number of procedures (which excludes the dummy + // library and kernel) + assert_eq!(program.num_procedures(), 3); } /// Ensures that a single copy of procedures with the same MAST root are added only once to the MAST @@ -276,48 +248,11 @@ fn explicit_fully_qualified_procedure_references() { exec.::foo::bar::bar end"#; - pub struct DummyLibrary { - namespace: LibraryNamespace, - #[allow(clippy::vec_box)] - modules: Vec>, - dependencies: Vec, - } - - impl Default for DummyLibrary { - fn default() -> Self { - let bar = - Module::parse_str(BAR_NAME.parse().unwrap(), ModuleKind::Library, BAR).unwrap(); - let baz = - Module::parse_str(BAZ_NAME.parse().unwrap(), ModuleKind::Library, BAZ).unwrap(); - let namespace = LibraryNamespace::new("foo").unwrap(); - Self { - namespace, - modules: vec![bar, baz], - dependencies: Vec::new(), - } - } - } - - impl Library for DummyLibrary { - fn root_ns(&self) -> &LibraryNamespace { - &self.namespace - } - - fn version(&self) -> &Version { - const MIN: Version = Version::min(); - &MIN - } - - fn modules(&self) -> impl ExactSizeIterator + '_ { - self.modules.iter().map(|m| m.as_ref()) - } - - fn dependencies(&self) -> &[LibraryNamespace] { - &self.dependencies - } - } - - let assembler = Assembler::default().with_library(&DummyLibrary::default()).unwrap(); + let bar = Module::parse_str(BAR_NAME.parse().unwrap(), ModuleKind::Library, BAR).unwrap(); + let baz = Module::parse_str(BAZ_NAME.parse().unwrap(), ModuleKind::Library, BAZ).unwrap(); + let library = Assembler::default().assemble_library([bar, baz]).unwrap(); + + let assembler = Assembler::default().with_compiled_library(&library).unwrap(); let program = r#" begin @@ -326,3 +261,40 @@ fn explicit_fully_qualified_procedure_references() { assert_matches!(assembler.assemble_program(program), Ok(_)); } + +#[test] +fn re_exports() { + const BAR_NAME: &str = "foo::bar"; + const BAR: &str = r#" + export.bar + add + end"#; + + const BAZ_NAME: &str = "foo::baz"; + const BAZ: &str = r#" + use.foo::bar + + export.bar::bar + + export.baz + push.1 push.2 add + end"#; + + let bar = Module::parse_str(BAR_NAME.parse().unwrap(), ModuleKind::Library, BAR).unwrap(); + let baz = Module::parse_str(BAZ_NAME.parse().unwrap(), ModuleKind::Library, BAZ).unwrap(); + let library = Assembler::default().assemble_library([bar, baz]).unwrap(); + + let assembler = Assembler::default().with_compiled_library(&library).unwrap(); + + let program = r#" + use.foo::baz + + begin + push.1 push.2 + exec.baz::baz + push.3 push.4 + exec.baz::bar + end"#; + + assert_matches!(assembler.assemble_program(program), Ok(_)); +} diff --git a/assembly/src/ast/invocation_target.rs b/assembly/src/ast/invocation_target.rs index 9871e0e012..52c85ca83b 100644 --- a/assembly/src/ast/invocation_target.rs +++ b/assembly/src/ast/invocation_target.rs @@ -16,6 +16,7 @@ pub enum InvokeKind { Exec = 0, Call, SysCall, + ProcRef, } /// Represents a specific invocation diff --git a/assembly/src/ast/module.rs b/assembly/src/ast/module.rs index bf31df4f24..b898228a87 100644 --- a/assembly/src/ast/module.rs +++ b/assembly/src/ast/module.rs @@ -7,7 +7,7 @@ use alloc::{ use core::fmt; use super::{ - Export, FullyQualifiedProcedureName, Import, LocalNameResolver, ProcedureIndex, ProcedureName, + Export, Import, LocalNameResolver, ProcedureIndex, ProcedureName, QualifiedProcedureName, ResolvedProcedure, }; use crate::{ @@ -374,7 +374,7 @@ impl Module { /// name. pub fn exported_procedures( &self, - ) -> impl Iterator + '_ { + ) -> impl Iterator + '_ { self.procedures.iter().enumerate().filter_map(|(proc_idx, p)| { // skip un-exported procedures if !p.visibility().is_exported() { @@ -382,7 +382,7 @@ impl Module { } let proc_idx = ProcedureIndex::new(proc_idx); - let fqn = FullyQualifiedProcedureName::new(self.path().clone(), p.name().clone()); + let fqn = QualifiedProcedureName::new(self.path().clone(), p.name().clone()); Some((proc_idx, fqn)) }) @@ -403,8 +403,9 @@ impl Module { /// Get an iterator over the "dependencies" of a module, i.e. what library namespaces we expect /// to find imported procedures in. /// - /// For example, if we have imported `std::math::u64`, then we would expect to import that - /// module from a [crate::Library] with the namespace `std`. + /// For example, if we have imported `std::math::u64`, then we would expect to find a library + /// on disk named `std.masl`, although that isn't a strict requirement. This notion of + /// dependencies may go away with future packaging-related changed. pub fn dependencies(&self) -> impl Iterator { self.import_paths().map(|import| import.namespace()) } @@ -444,7 +445,9 @@ impl Module { } Export::Alias(ref alias) => match alias.target() { AliasTarget::MastRoot(digest) => Some(ResolvedProcedure::MastRoot(**digest)), - AliasTarget::Path(path) => Some(ResolvedProcedure::External(path.clone())), + AliasTarget::ProcedurePath(path) | AliasTarget::AbsoluteProcedurePath(path) => { + Some(ResolvedProcedure::External(path.clone())) + } }, } } @@ -457,9 +460,11 @@ impl Module { ResolvedProcedure::Local(Span::new(p.name().span(), ProcedureIndex::new(i))), ), Export::Alias(ref p) => { - let target = match p.target { - AliasTarget::MastRoot(ref digest) => ResolvedProcedure::MastRoot(**digest), - AliasTarget::Path(ref path) => ResolvedProcedure::External(path.clone()), + let target = match p.target() { + AliasTarget::MastRoot(digest) => ResolvedProcedure::MastRoot(**digest), + AliasTarget::ProcedurePath(path) | AliasTarget::AbsoluteProcedurePath(path) => { + ResolvedProcedure::External(path.clone()) + } }; (p.name().clone(), target) } diff --git a/assembly/src/ast/procedure/alias.rs b/assembly/src/ast/procedure/alias.rs index c1661163f8..1a7b58ce0f 100644 --- a/assembly/src/ast/procedure/alias.rs +++ b/assembly/src/ast/procedure/alias.rs @@ -1,10 +1,11 @@ use alloc::{string::String, sync::Arc}; use core::fmt; -use super::{FullyQualifiedProcedureName, ProcedureName}; +use super::{ProcedureName, QualifiedProcedureName}; use crate::{ - ast::AstSerdeOptions, diagnostics::SourceFile, ByteReader, ByteWriter, DeserializationError, - RpoDigest, SourceSpan, Span, Spanned, + ast::{AstSerdeOptions, InvocationTarget}, + diagnostics::SourceFile, + ByteReader, ByteWriter, DeserializationError, RpoDigest, SourceSpan, Span, Spanned, }; // PROCEDURE ALIAS @@ -22,29 +23,24 @@ pub struct ProcedureAlias { source_file: Option>, /// The documentation attached to this procedure docs: Option>, - /// The name of the re-exported procedure. + /// The name of this procedure name: ProcedureName, - /// The fully-qualified name of the imported procedure + /// The underlying procedure being aliased. /// - /// NOTE: This is fully-qualified from the perspective of the containing [Module], but may not - /// be fully-resolved to the concrete definition until compilation time. - pub(crate) target: AliasTarget, - /// If true, this alias was created with an absolute path, bypassing the need for an import - absolute: bool, + /// Alias targets are context-sensitive, depending on how they were defined and what stage of + /// compilation we're in. See [AliasTarget] for semantics of each target type, but they closely + /// correspond to [InvocationTarget]. + target: AliasTarget, } impl ProcedureAlias { /// Creates a new procedure alias called `name`, which resolves to `target`. - pub fn new(name: ProcedureName, target: impl Into, absolute: bool) -> Self { - let target = target.into(); - // Ignore the absolute flag if the target is implicitly absolute - let absolute = matches!(target, AliasTarget::MastRoot(_)) || absolute; + pub fn new(name: ProcedureName, target: AliasTarget) -> Self { Self { docs: None, source_file: None, name, target, - absolute, } } @@ -74,19 +70,38 @@ impl ProcedureAlias { /// /// If the procedure is simply re-exported with the same name, this will be equivalent to /// `self.target().name` + #[inline] pub fn name(&self) -> &ProcedureName { &self.name } /// Returns the target of this procedure alias + #[inline] pub fn target(&self) -> &AliasTarget { &self.target } + /// Returns a mutable reference to the target of this procedure alias + #[inline] + pub fn target_mut(&mut self) -> &mut AliasTarget { + &mut self.target + } + /// Returns true if this procedure uses an absolute target path #[inline] - pub fn absolute(&self) -> bool { - self.absolute + pub fn is_absolute(&self) -> bool { + matches!(self.target, AliasTarget::MastRoot(_) | AliasTarget::AbsoluteProcedurePath(_)) + } + + /// Returns true if this alias uses a different name than the target procedure + #[inline] + pub fn is_renamed(&self) -> bool { + match self.target() { + AliasTarget::MastRoot(_) => true, + AliasTarget::ProcedurePath(fqn) | AliasTarget::AbsoluteProcedurePath(fqn) => { + fqn.name != self.name + } + } } } @@ -95,7 +110,6 @@ impl ProcedureAlias { pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { self.name.write_into_with_options(target, options); self.target.write_into_with_options(target, options); - target.write_bool(self.absolute); } pub fn read_from_with_options( @@ -104,13 +118,11 @@ impl ProcedureAlias { ) -> Result { let name = ProcedureName::read_from_with_options(source, options)?; let target = AliasTarget::read_from_with_options(source, options)?; - let absolute = source.read_bool()?; Ok(Self { source_file: None, docs: None, name, target, - absolute, }) } } @@ -137,12 +149,12 @@ impl crate::prettier::PrettyPrint for ProcedureAlias { doc += const_text("export."); doc += match &self.target { target @ AliasTarget::MastRoot(_) => display(format_args!("{}->{}", target, self.name)), - AliasTarget::Path(name) => { - let prefix = if self.absolute { "::" } else { "" }; - if name.name == self.name { - display(format_args!("{}{}", prefix, name)) + target => { + let prefix = if self.is_absolute() { "::" } else { "" }; + if self.is_renamed() { + display(format_args!("{}{}->{}", prefix, target, &self.name)) } else { - display(format_args!("{}{}->{}", prefix, name, &self.name)) + display(format_args!("{}{}", prefix, target)) } } }; @@ -152,18 +164,27 @@ impl crate::prettier::PrettyPrint for ProcedureAlias { /// A fully-qualified external procedure that is the target of a procedure alias #[derive(Debug, Clone, PartialEq, Eq)] +#[repr(u8)] pub enum AliasTarget { - /// An alias of a procedure with the given digest + /// An alias of the procedure whose root is the given digest + /// + /// Corresponds to [`InvocationTarget::MastRoot`] MastRoot(Span), - /// An alias of a procedure with the given fully-qualified path - Path(FullyQualifiedProcedureName), + /// An alias of `name`, imported from `module` + /// + /// Corresponds to [`InvocationTarget::ProcedurePath`] + ProcedurePath(QualifiedProcedureName), + /// An alias of a procedure with the given absolute, fully-qualified path + /// + /// Corresponds to [InvocationTarget::AbsoluteProcedurePath] + AbsoluteProcedurePath(QualifiedProcedureName), } impl Spanned for AliasTarget { fn span(&self) -> SourceSpan { match self { Self::MastRoot(spanned) => spanned.span(), - Self::Path(spanned) => spanned.span(), + Self::ProcedurePath(spanned) | Self::AbsoluteProcedurePath(spanned) => spanned.span(), } } } @@ -174,9 +195,56 @@ impl From> for AliasTarget { } } -impl From for AliasTarget { - fn from(path: FullyQualifiedProcedureName) -> Self { - Self::Path(path) +impl TryFrom for AliasTarget { + type Error = InvocationTarget; + + fn try_from(target: InvocationTarget) -> Result { + let span = target.span(); + match target { + InvocationTarget::MastRoot(digest) => Ok(Self::MastRoot(digest)), + InvocationTarget::ProcedurePath { name, module } => { + let ns = crate::LibraryNamespace::from_ident_unchecked(module); + let module = crate::LibraryPath::new_from_components(ns, []); + Ok(Self::ProcedurePath(QualifiedProcedureName { span, module, name })) + } + InvocationTarget::AbsoluteProcedurePath { name, path: module } => { + Ok(Self::AbsoluteProcedurePath(QualifiedProcedureName { span, module, name })) + } + target @ InvocationTarget::ProcedureName(_) => Err(target), + } + } +} + +impl From<&AliasTarget> for InvocationTarget { + fn from(target: &AliasTarget) -> Self { + match target { + AliasTarget::MastRoot(digest) => Self::MastRoot(*digest), + AliasTarget::ProcedurePath(ref fqn) => { + let name = fqn.name.clone(); + let module = fqn.module.last_component().to_ident(); + Self::ProcedurePath { name, module } + } + AliasTarget::AbsoluteProcedurePath(ref fqn) => Self::AbsoluteProcedurePath { + name: fqn.name.clone(), + path: fqn.module.clone(), + }, + } + } +} +impl From for InvocationTarget { + fn from(target: AliasTarget) -> Self { + match target { + AliasTarget::MastRoot(digest) => Self::MastRoot(digest), + AliasTarget::ProcedurePath(fqn) => { + let name = fqn.name; + let module = fqn.module.last_component().to_ident(); + Self::ProcedurePath { name, module } + } + AliasTarget::AbsoluteProcedurePath(fqn) => Self::AbsoluteProcedurePath { + name: fqn.name, + path: fqn.module, + }, + } } } @@ -187,7 +255,8 @@ impl crate::prettier::PrettyPrint for AliasTarget { match self { Self::MastRoot(digest) => display(DisplayHex(digest.as_bytes().as_slice())), - Self::Path(path) => display(format_args!("{}", path)), + Self::ProcedurePath(fqn) => display(fqn), + Self::AbsoluteProcedurePath(fqn) => display(format_args!("::{}", fqn)), } } } @@ -215,7 +284,8 @@ impl AliasTarget { target.write_u8(self.tag()); match self { Self::MastRoot(spanned) => spanned.write_into(target, options), - Self::Path(path) => path.write_into_with_options(target, options), + Self::ProcedurePath(path) => path.write_into_with_options(target, options), + Self::AbsoluteProcedurePath(path) => path.write_into_with_options(target, options), } } @@ -230,8 +300,12 @@ impl AliasTarget { Ok(Self::MastRoot(root)) } 1 => { - let path = FullyQualifiedProcedureName::read_from_with_options(source, options)?; - Ok(Self::Path(path)) + let path = QualifiedProcedureName::read_from_with_options(source, options)?; + Ok(Self::ProcedurePath(path)) + } + 2 => { + let path = QualifiedProcedureName::read_from_with_options(source, options)?; + Ok(Self::AbsoluteProcedurePath(path)) } n => Err(DeserializationError::InvalidValue(format!( "{} is not a valid alias target type", diff --git a/assembly/src/ast/procedure/mod.rs b/assembly/src/ast/procedure/mod.rs index 72b48dff43..b6c03730d2 100644 --- a/assembly/src/ast/procedure/mod.rs +++ b/assembly/src/ast/procedure/mod.rs @@ -7,7 +7,7 @@ mod resolver; pub use self::alias::{AliasTarget, ProcedureAlias}; pub use self::id::ProcedureIndex; -pub use self::name::{FullyQualifiedProcedureName, ProcedureName}; +pub use self::name::{ProcedureName, QualifiedProcedureName}; pub use self::procedure::{Procedure, Visibility}; pub use self::resolver::{LocalNameResolver, ResolvedProcedure}; diff --git a/assembly/src/ast/procedure/name.rs b/assembly/src/ast/procedure/name.rs index 0e893129ee..5fc1e18e6c 100644 --- a/assembly/src/ast/procedure/name.rs +++ b/assembly/src/ast/procedure/name.rs @@ -8,17 +8,20 @@ use core::{ use crate::{ ast::{AstSerdeOptions, CaseKindError, Ident, IdentError}, diagnostics::{IntoDiagnostic, Report}, - ByteReader, ByteWriter, Deserializable, DeserializationError, LibraryPath, Serializable, - SourceSpan, Span, Spanned, + ByteReader, ByteWriter, Deserializable, DeserializationError, LibraryNamespace, LibraryPath, + Serializable, SourceSpan, Span, Spanned, }; -// FULLY QUALIFIED PROCEDURE NAME +// QUALIFIED PROCEDURE NAME // ================================================================================================ -/// Represents a fully-qualified procedure name, e.g. `std::math::u64::add`, parsed into it's +/// Represents a qualified procedure name, e.g. `std::math::u64::add`, parsed into it's /// constituent [LibraryPath] and [ProcedureName] components. +/// +/// A qualified procedure name can be context-sensitive, i.e. the module path might refer +/// to an imported #[derive(Clone)] -pub struct FullyQualifiedProcedureName { +pub struct QualifiedProcedureName { /// The source span associated with this identifier. pub span: SourceSpan, /// The module path for this procedure. @@ -27,8 +30,8 @@ pub struct FullyQualifiedProcedureName { pub name: ProcedureName, } -impl FullyQualifiedProcedureName { - /// Create a new [FullyQualifiedProcedureName] with the given fully-qualified module path +impl QualifiedProcedureName { + /// Create a new [QualifiedProcedureName] with the given fully-qualified module path /// and procedure name. pub fn new(module: LibraryPath, name: ProcedureName) -> Self { Self { @@ -37,9 +40,14 @@ impl FullyQualifiedProcedureName { name, } } + + /// Returns the namespace of this fully-qualified procedure name. + pub fn namespace(&self) -> &LibraryNamespace { + self.module.namespace() + } } -impl FromStr for FullyQualifiedProcedureName { +impl FromStr for QualifiedProcedureName { type Err = Report; fn from_str(s: &str) -> Result { @@ -54,39 +62,39 @@ impl FromStr for FullyQualifiedProcedureName { } } -impl Eq for FullyQualifiedProcedureName {} +impl Eq for QualifiedProcedureName {} -impl PartialEq for FullyQualifiedProcedureName { +impl PartialEq for QualifiedProcedureName { fn eq(&self, other: &Self) -> bool { self.name == other.name && self.module == other.module } } -impl Ord for FullyQualifiedProcedureName { +impl Ord for QualifiedProcedureName { fn cmp(&self, other: &Self) -> core::cmp::Ordering { self.module.cmp(&other.module).then_with(|| self.name.cmp(&other.name)) } } -impl PartialOrd for FullyQualifiedProcedureName { +impl PartialOrd for QualifiedProcedureName { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl From for miette::SourceSpan { - fn from(fqn: FullyQualifiedProcedureName) -> Self { +impl From for miette::SourceSpan { + fn from(fqn: QualifiedProcedureName) -> Self { fqn.span.into() } } -impl Spanned for FullyQualifiedProcedureName { +impl Spanned for QualifiedProcedureName { fn span(&self) -> SourceSpan { self.span } } -impl fmt::Debug for FullyQualifiedProcedureName { +impl fmt::Debug for QualifiedProcedureName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { f.debug_struct("FullyQualifiedProcedureName") .field("module", &self.module) @@ -95,14 +103,22 @@ impl fmt::Debug for FullyQualifiedProcedureName { } } -impl fmt::Display for FullyQualifiedProcedureName { +impl crate::prettier::PrettyPrint for QualifiedProcedureName { + fn render(&self) -> vm_core::prettier::Document { + use crate::prettier::*; + + display(self) + } +} + +impl fmt::Display for QualifiedProcedureName { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!(f, "{}::{}", &self.module, &self.name) } } /// Serialization -impl FullyQualifiedProcedureName { +impl QualifiedProcedureName { /// Serialize to `target` using `options` pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { if options.debug_info { diff --git a/assembly/src/ast/procedure/resolver.rs b/assembly/src/ast/procedure/resolver.rs index 8199b32d68..27cad535d3 100644 --- a/assembly/src/ast/procedure/resolver.rs +++ b/assembly/src/ast/procedure/resolver.rs @@ -1,4 +1,4 @@ -use super::{FullyQualifiedProcedureName, ProcedureIndex, ProcedureName}; +use super::{ProcedureIndex, ProcedureName, QualifiedProcedureName}; use crate::{ast::Ident, LibraryPath, RpoDigest, SourceSpan, Span, Spanned}; use alloc::{collections::BTreeMap, vec::Vec}; @@ -11,7 +11,7 @@ pub enum ResolvedProcedure { /// The name was resolved to a procedure definition in the same module at the given index Local(Span), /// The name was resolved to a procedure exported from another module - External(FullyQualifiedProcedureName), + External(QualifiedProcedureName), /// The name was resolved to a procedure with a known MAST root MastRoot(RpoDigest), } diff --git a/assembly/src/ast/visit.rs b/assembly/src/ast/visit.rs index 22a2e1411d..c60eebeb57 100644 --- a/assembly/src/ast/visit.rs +++ b/assembly/src/ast/visit.rs @@ -119,6 +119,9 @@ pub trait Visit { fn visit_invoke_target(&mut self, target: &InvocationTarget) -> ControlFlow { visit_invoke_target(self, target) } + fn visit_alias_target(&mut self, target: &AliasTarget) -> ControlFlow { + visit_alias_target(self, target) + } fn visit_immediate_u8(&mut self, imm: &Immediate) -> ControlFlow { visit_immediate_u8(self, imm) } @@ -185,6 +188,9 @@ where fn visit_invoke_target(&mut self, target: &InvocationTarget) -> ControlFlow { (**self).visit_invoke_target(target) } + fn visit_alias_target(&mut self, target: &AliasTarget) -> ControlFlow { + (**self).visit_alias_target(target) + } fn visit_immediate_u8(&mut self, imm: &Immediate) -> ControlFlow { (**self).visit_immediate_u8(imm) } @@ -242,11 +248,11 @@ where } #[inline(always)] -pub fn visit_procedure_alias(_visitor: &mut V, _alias: &ProcedureAlias) -> ControlFlow +pub fn visit_procedure_alias(visitor: &mut V, alias: &ProcedureAlias) -> ControlFlow where V: ?Sized + Visit, { - ControlFlow::Continue(()) + visitor.visit_alias_target(alias.target()) } pub fn visit_block(visitor: &mut V, block: &Block) -> ControlFlow @@ -433,6 +439,14 @@ where ControlFlow::Continue(()) } +#[inline(always)] +pub fn visit_alias_target(_visitor: &mut V, _target: &AliasTarget) -> ControlFlow +where + V: ?Sized + Visit, +{ + ControlFlow::Continue(()) +} + #[inline(always)] pub fn visit_immediate_u8(_visitor: &mut V, _imm: &Immediate) -> ControlFlow where @@ -545,6 +559,9 @@ pub trait VisitMut { fn visit_mut_invoke_target(&mut self, target: &mut InvocationTarget) -> ControlFlow { visit_mut_invoke_target(self, target) } + fn visit_mut_alias_target(&mut self, target: &mut AliasTarget) -> ControlFlow { + visit_mut_alias_target(self, target) + } fn visit_mut_immediate_u8(&mut self, imm: &mut Immediate) -> ControlFlow { visit_mut_immediate_u8(self, imm) } @@ -614,6 +631,9 @@ where fn visit_mut_invoke_target(&mut self, target: &mut InvocationTarget) -> ControlFlow { (**self).visit_mut_invoke_target(target) } + fn visit_mut_alias_target(&mut self, target: &mut AliasTarget) -> ControlFlow { + (**self).visit_mut_alias_target(target) + } fn visit_mut_immediate_u8(&mut self, imm: &mut Immediate) -> ControlFlow { (**self).visit_mut_immediate_u8(imm) } @@ -672,13 +692,13 @@ where #[inline(always)] pub fn visit_mut_procedure_alias( - _visitor: &mut V, - _alias: &mut ProcedureAlias, + visitor: &mut V, + alias: &mut ProcedureAlias, ) -> ControlFlow where V: ?Sized + VisitMut, { - ControlFlow::Continue(()) + visitor.visit_mut_alias_target(alias.target_mut()) } pub fn visit_mut_block(visitor: &mut V, block: &mut Block) -> ControlFlow @@ -885,6 +905,14 @@ where ControlFlow::Continue(()) } +#[inline(always)] +pub fn visit_mut_alias_target(_visitor: &mut V, _target: &mut AliasTarget) -> ControlFlow +where + V: ?Sized + VisitMut, +{ + ControlFlow::Continue(()) +} + #[inline(always)] pub fn visit_mut_immediate_u8(_visitor: &mut V, _imm: &mut Immediate) -> ControlFlow where diff --git a/assembly/src/compile.rs b/assembly/src/compile.rs index d40577d2b3..359ee1bfb4 100644 --- a/assembly/src/compile.rs +++ b/assembly/src/compile.rs @@ -84,7 +84,7 @@ impl Options { /// This trait is meant to be implemented by any type that can be compiled to a [Module], /// to allow methods which expect a [Module] to accept things like: /// -/// * A [Module] which was previously compiled or read from a [crate::Library]. +/// * A [Module] which was previously parsed or deserialized /// * A string representing the source code of a [Module]. /// * A path to a file containing the source code of a [Module]. /// * A vector of [crate::ast::Form]s comprising the contents of a [Module]. diff --git a/assembly/src/errors.rs b/assembly/src/errors.rs index d55d73bbec..da3bb78153 100644 --- a/assembly/src/errors.rs +++ b/assembly/src/errors.rs @@ -2,9 +2,9 @@ use alloc::{string::String, sync::Arc, vec::Vec}; use vm_core::mast::MastForestError; use crate::{ - ast::FullyQualifiedProcedureName, + ast::QualifiedProcedureName, diagnostics::{Diagnostic, RelatedError, RelatedLabel, Report, SourceFile}, - LibraryNamespace, LibraryPath, RpoDigest, SourceSpan, + LibraryNamespace, LibraryPath, SourceSpan, }; // ASSEMBLY ERROR @@ -30,8 +30,8 @@ pub enum AssemblyError { #[error("two procedures found with same mast root, but conflicting definitions ('{first}' and '{second}')")] #[diagnostic()] ConflictingDefinitions { - first: FullyQualifiedProcedureName, - second: FullyQualifiedProcedureName, + first: QualifiedProcedureName, + second: QualifiedProcedureName, }, #[error("duplicate definition found for module '{path}'")] #[diagnostic()] @@ -58,16 +58,7 @@ pub enum AssemblyError { span: SourceSpan, #[source_code] source_file: Option>, - callee: FullyQualifiedProcedureName, - }, - #[error("invalid syscall: kernel procedures must be available during compilation, but '{callee}' is not")] - #[diagnostic()] - UnknownSysCallTarget { - #[label("call occurs here")] - span: SourceSpan, - #[source_code] - source_file: Option>, - callee: RpoDigest, + callee: QualifiedProcedureName, }, #[error("invalid use of 'caller' instruction outside of kernel")] #[diagnostic(help( diff --git a/assembly/src/lib.rs b/assembly/src/lib.rs index c0a3e124cf..09c4075ffa 100644 --- a/assembly/src/lib.rs +++ b/assembly/src/lib.rs @@ -33,9 +33,7 @@ mod tests; pub use self::assembler::Assembler; pub use self::compile::{Compile, Options as CompileOptions}; pub use self::errors::{AssemblyError, CompiledLibraryError}; -pub use self::library::{ - Library, LibraryError, LibraryNamespace, LibraryPath, MaslLibrary, PathError, Version, -}; +pub use self::library::{LibraryError, LibraryNamespace, LibraryPath, PathError, Version}; pub use self::parser::{ModuleParser, SourceLocation, SourceSpan, Span, Spanned}; /// Re-exported for downstream crates diff --git a/assembly/src/library/error.rs b/assembly/src/library/error.rs index b8361d04b4..3a9afb1c57 100644 --- a/assembly/src/library/error.rs +++ b/assembly/src/library/error.rs @@ -1,10 +1,11 @@ -use alloc::string::String; +use alloc::{string::String, vec::Vec}; use vm_core::errors::KernelError; use crate::{ - ast::FullyQualifiedProcedureName, + ast::QualifiedProcedureName, diagnostics::Diagnostic, library::{LibraryNamespaceError, VersionError}, + prettier::pretty_print_csv, DeserializationError, LibraryNamespace, LibraryPath, PathError, }; @@ -65,16 +66,19 @@ pub enum LibraryError { #[derive(Debug, thiserror::Error, Diagnostic)] pub enum CompiledLibraryError { - #[error("Invalid exports: MAST forest has {roots_len} procedure roots, but exports have {exports_len}")] + #[error("Invalid exports: there must be at least one export")] #[diagnostic()] - InvalidExports { - exports_len: usize, - roots_len: usize, - }, - #[error("Invalid export in kernel library: {procedure_path}")] + EmptyExports, + #[error("exports are not in the same namespace; all namespaces: {namespaces:?}")] + InconsistentNamespaces { namespaces: Vec }, + #[error("invalid export in kernel library: {procedure_path}")] InvalidKernelExport { - procedure_path: FullyQualifiedProcedureName, + procedure_path: QualifiedProcedureName, }, #[error(transparent)] Kernel(#[from] KernelError), + #[error("no MAST roots for the following exports: {}", pretty_print_csv(missing_exports.as_slice()))] + MissingExports { + missing_exports: Vec, + }, } diff --git a/assembly/src/library/masl.rs b/assembly/src/library/masl.rs index 898b9b5799..d7ba54c827 100644 --- a/assembly/src/library/masl.rs +++ b/assembly/src/library/masl.rs @@ -1,285 +1,18 @@ -use super::{Library, LibraryNamespace, Version, MAX_DEPENDENCIES, MAX_MODULES}; -use crate::{ - ast::{self, AstSerdeOptions}, - ByteReader, ByteWriter, Deserializable, DeserializationError, LibraryError, Serializable, -}; -use alloc::{collections::BTreeSet, sync::Arc, vec::Vec}; +use super::*; -/// Serialization options for [ModuleAst]. Imports and information about imported procedures are -/// part of the ModuleAst serialization by default. -const AST_DEFAULT_SERDE_OPTIONS: AstSerdeOptions = AstSerdeOptions { - serialize_imports: true, - debug_info: true, -}; - -/// A concrete implementation of the Library trait. Contains the minimal attributes of a functional -/// library. -/// -/// Implementers of the library trait should use this base type to perform serialization into -/// `masl` files. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MaslLibrary { - /// Root namespace of the library. - namespace: LibraryNamespace, - /// Version of the library. - version: Version, - /// Available modules. - modules: Vec>, - /// Dependencies of the library. - dependencies: Vec, -} - -impl Library for MaslLibrary { - fn root_ns(&self) -> &LibraryNamespace { - &self.namespace - } - - fn version(&self) -> &Version { - &self.version - } - - fn modules(&self) -> impl ExactSizeIterator + '_ { - self.modules.iter().map(|m| m.as_ref()) - } - - fn dependencies(&self) -> &[LibraryNamespace] { - &self.dependencies - } -} - -impl MaslLibrary { - /// File extension for the Assembly Library. - pub const LIBRARY_EXTENSION: &'static str = "masl"; - - /// File extension for the Assembly Module. - pub const MODULE_EXTENSION: &'static str = "masm"; - - /// Name of the root module. - pub const MOD: &'static str = "mod"; - - /// Returns a new [Library] instantiated from the specified parameters. - /// - /// # Errors - /// Returns an error if the provided `modules` vector is empty or contains more than - /// [u16::MAX] elements. - pub fn new( - namespace: LibraryNamespace, - version: Version, - modules: I, - dependencies: Vec, - ) -> Result - where - I: IntoIterator, - Arc: From, - { - let modules = modules.into_iter().map(Arc::from).collect::>(); - let library = Self { - namespace, - version, - modules, - dependencies, - }; - - library.validate()?; - - Ok(library) - } - - fn validate(&self) -> Result<(), LibraryError> { - if self.modules.is_empty() { - return Err(LibraryError::Empty(self.namespace.clone())); - } - if self.modules.len() > MAX_MODULES { - return Err(LibraryError::TooManyModulesInLibrary { - name: self.namespace.clone(), - count: self.modules.len(), - max: MAX_MODULES, - }); - } - - if self.dependencies.len() > MAX_DEPENDENCIES { - return Err(LibraryError::TooManyDependenciesInLibrary { - name: self.namespace.clone(), - count: self.dependencies.len(), - max: MAX_DEPENDENCIES, - }); - } - - Ok(()) - } +pub struct LibraryEntry { + pub name: LibraryPath, + pub source_path: std::path::PathBuf, } -#[cfg(feature = "std")] -mod use_std { - use std::{collections::BTreeMap, fs, io, path::Path}; - - use super::*; - use crate::{ - ast::ModuleKind, - diagnostics::{tracing::instrument, IntoDiagnostic, Report}, - }; - - impl MaslLibrary { - /// Read a directory and recursively create modules from its `masm` files. - /// - /// For every directory, concatenate the module path with the dir name and proceed. - /// - /// For every file, pick and parse the ones with `masm` extension; skip otherwise. - /// - /// Example: - /// - /// - ./sys.masm -> ("sys", ast(./sys.masm)) - /// - ./crypto/hash.masm -> ("crypto::hash", ast(./crypto/hash.masm)) - /// - ./math/u32.masm -> ("math::u32", ast(./math/u32.masm)) - /// - ./math/u64.masm -> ("math::u64", ast(./math/u64.masm)) - pub fn read_from_dir

( - path: P, - namespace: LibraryNamespace, - version: Version, - ) -> Result - where - P: AsRef, - { - let path = path.as_ref(); - if !path.is_dir() { - return Err(Report::msg(format!( - "the provided path '{}' is not a valid directory", - path.display() - ))); - } - - // mod.masm is not allowed in the root directory - if path.join("mod.masm").exists() { - return Err(Report::msg("mod.masm is not allowed in the root directory")); - } - - let library = Self { - namespace, - version, - modules: Default::default(), - dependencies: Default::default(), - }; - library.load_modules_from_dir(path) - } - - /// Read a library from a file. - #[instrument(name = "read_library_file", fields(path = %path.as_ref().display()))] - pub fn read_from_file

(path: P) -> Result - where - P: AsRef, - { - use vm_core::utils::ReadAdapter; - - let path = path.as_ref(); - let mut file = fs::File::open(path)?; - let mut adapter = ReadAdapter::new(&mut file); - ::read_from(&mut adapter).map_err(|error| { - LibraryError::DeserializationFailed { - path: path.to_string_lossy().into_owned(), - error, - } - }) - } - - /// Write the library to a target director, using its namespace as file name and the - /// appropriate extension. - pub fn write_to_dir

(&self, dir_path: P) -> io::Result<()> - where - P: AsRef, - { - fs::create_dir_all(&dir_path)?; - let mut path = dir_path.as_ref().join(self.namespace.as_ref()); - path.set_extension(Self::LIBRARY_EXTENSION); - - // NOTE: We catch panics due to i/o errors here due to the fact - // that the ByteWriter trait does not provide fallible APIs, so - // WriteAdapter will panic if the underlying writes fail. This - // needs to be addressed in winterfall at some point - std::panic::catch_unwind(|| { - let mut file = fs::File::create(path)?; - self.write_into(&mut file); - Ok(()) - }) - .map_err(|p| { - match p.downcast::() { - // SAFETY: It is guaranteed safe to read Box - Ok(err) => unsafe { core::ptr::read(&*err) }, - Err(err) => std::panic::resume_unwind(err), - } - })? - } - - /// Read the contents (modules) of this library from `dir`, returning any errors that occur - /// while traversing the file system. - /// - /// Errors may also be returned if traversal discovers issues with the library, such as - /// invalid names, etc. - /// - /// Returns the set of modules that were parsed - fn load_modules_from_dir(mut self, dir: &Path) -> Result { - use crate::diagnostics::WrapErr; - use alloc::collections::btree_map::Entry; - - let mut modules = BTreeMap::default(); - let mut dependencies = BTreeSet::default(); - - let walker = WalkLibrary::new(self.namespace.clone(), dir) - .into_diagnostic() - .wrap_err_with(|| format!("failed to load library from '{}'", dir.display()))?; - for entry in walker { - let LibraryEntry { - mut name, - source_path, - } = entry?; - if name.last() == MaslLibrary::MOD { - name.pop(); - } - // Parse module at the given path - let ast = ast::Module::parse_file(name.clone(), ModuleKind::Library, &source_path)?; - // Add dependencies of this module to the global set - for path in ast.import_paths() { - let ns = path.namespace(); - if ns != &self.namespace { - dependencies.insert(ns.clone()); - } - } - match modules.entry(name) { - Entry::Occupied(ref entry) => { - return Err(LibraryError::DuplicateModulePath(entry.key().clone())) - .into_diagnostic(); - } - Entry::Vacant(entry) => { - entry.insert(Arc::from(ast)); - } - } - } - - self.modules.extend(modules.into_values()); - self.dependencies.extend(dependencies); - - self.validate()?; - - Ok(self) - } - } -} - -#[cfg(feature = "std")] -struct LibraryEntry { - name: super::LibraryPath, - source_path: std::path::PathBuf, -} - -#[cfg(feature = "std")] -struct WalkLibrary<'a> { +pub struct WalkLibrary<'a> { namespace: LibraryNamespace, root: &'a std::path::Path, stack: alloc::collections::VecDeque>, } -#[cfg(feature = "std")] impl<'a> WalkLibrary<'a> { - fn new(namespace: LibraryNamespace, path: &'a std::path::Path) -> std::io::Result { + pub fn new(namespace: LibraryNamespace, path: &'a std::path::Path) -> std::io::Result { use alloc::collections::VecDeque; let stack = VecDeque::from_iter(std::fs::read_dir(path)?); @@ -311,7 +44,7 @@ impl<'a> WalkLibrary<'a> { let mut file_path = entry.path(); let is_module = file_path .extension() - .map(|ext| ext == AsRef::::as_ref(MaslLibrary::MODULE_EXTENSION)) + .map(|ext| ext == AsRef::::as_ref(CompiledLibrary::MODULE_EXTENSION)) .unwrap_or(false); if !is_module { return Ok(None); @@ -346,7 +79,6 @@ impl<'a> WalkLibrary<'a> { } } -#[cfg(feature = "std")] impl<'a> Iterator for WalkLibrary<'a> { type Item = Result; fn next(&mut self) -> Option { @@ -370,58 +102,3 @@ impl<'a> Iterator for WalkLibrary<'a> { } } } - -impl Serializable for MaslLibrary { - #[inline] - fn write_into(&self, target: &mut W) { - self.write_into_with_options(target, AST_DEFAULT_SERDE_OPTIONS) - } -} - -/// Serialization -impl MaslLibrary { - pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { - self.namespace.write_into(target); - self.version.write_into(target); - - let modules = self.modules(); - - // write dependencies - target.write_u16(self.dependencies.len() as u16); - self.dependencies.iter().for_each(|dep| dep.write_into(target)); - - // this assert is OK because maximum number of modules is enforced by Library constructor - debug_assert!(modules.len() <= MAX_MODULES, "too many modules"); - - target.write_u16(modules.len() as u16); - modules.for_each(|module| { - module.write_into_with_options(target, options); - }); - } -} - -impl Deserializable for MaslLibrary { - fn read_from(source: &mut R) -> Result { - let namespace = LibraryNamespace::read_from(source)?; - let version = Version::read_from(source)?; - - // read dependencies - let num_deps = source.read_u16()? as usize; - // TODO: check for duplicate/self-referential dependencies? - let deps_set: BTreeSet = (0..num_deps) - .map(|_| LibraryNamespace::read_from(source)) - .collect::>()?; - - // read modules - let num_modules = source.read_u16()? as usize; - let mut modules = Vec::with_capacity(num_modules); - for _ in 0..num_modules { - let ast = ast::Module::read_from(source)?; - modules.push(ast); - } - - let deps = deps_set.into_iter().collect(); - Self::new(namespace, version, modules, deps) - .map_err(|err| DeserializationError::InvalidValue(format!("{err}"))) - } -} diff --git a/assembly/src/library/mod.rs b/assembly/src/library/mod.rs index fcaf60f6e4..32cc3e882f 100644 --- a/assembly/src/library/mod.rs +++ b/assembly/src/library/mod.rs @@ -1,19 +1,23 @@ -use alloc::{collections::BTreeMap, vec::Vec}; +use alloc::{ + collections::{BTreeMap, BTreeSet}, + vec::Vec, +}; use vm_core::crypto::hash::RpoDigest; -use vm_core::mast::MastForest; +use vm_core::mast::{MastForest, MastNodeId}; +use vm_core::utils::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable}; use vm_core::Kernel; -use crate::ast::{self, FullyQualifiedProcedureName, ProcedureIndex, ProcedureName}; +use crate::ast::{AstSerdeOptions, ProcedureIndex, ProcedureName, QualifiedProcedureName}; mod error; +#[cfg(feature = "std")] mod masl; mod namespace; mod path; mod version; pub use self::error::{CompiledLibraryError, LibraryError}; -pub use self::masl::MaslLibrary; pub use self::namespace::{LibraryNamespace, LibraryNamespaceError}; pub use self::path::{LibraryPath, LibraryPathComponent, PathError}; pub use self::version::{Version, VersionError}; @@ -24,55 +28,109 @@ mod tests; // COMPILED LIBRARY // ================================================================================================ -/// Represents a library where all modules modules were compiled into a [`MastForest`]. +/// Represents a library where all modules were compiled into a [`MastForest`]. +/// +/// A library exports a set of one or more procedures. Currently, all exported procedures belong +/// to the same top-level namespace. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct CompiledLibrary { + /// The content hash of this library, formed by hashing the roots of all exports in + /// lexicographical order (by digest, not procedure name) + digest: RpoDigest, + /// A map between procedure paths and the corresponding procedure toots in the MAST forest. + /// Multiple paths can map to the same root, and also, some roots may not be associated with + /// any paths. + exports: BTreeMap, + /// The MAST forest underlying this library. mast_forest: MastForest, - // a path for every `root` in the associated MAST forest - exports: Vec, +} + +impl AsRef for CompiledLibrary { + #[inline(always)] + fn as_ref(&self) -> &CompiledLibrary { + self + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[repr(u8)] +enum Export { + /// The export is contained in the [MastForest] of this library + Local(MastNodeId), + /// The export is a re-export of an externally-defined procedure from another library + External(RpoDigest), } /// Constructors impl CompiledLibrary { - /// Constructs a new [`CompiledLibrary`]. + /// Constructs a new [`CompiledLibrary`] from the provided MAST forest and a set of exports. + /// + /// # Errors + /// + /// Returns an error if: + /// - The set of exported procedures is empty. + /// - Not all exported procedures are present in the MAST forest. pub fn new( mast_forest: MastForest, - exports: Vec, + exports: BTreeMap, ) -> Result { - if mast_forest.num_procedures() as usize != exports.len() { - return Err(CompiledLibraryError::InvalidExports { - exports_len: exports.len(), - roots_len: mast_forest.num_procedures() as usize, - }); + if exports.is_empty() { + return Err(CompiledLibraryError::EmptyExports); } + let mut fqn_to_export = BTreeMap::new(); + + // convert fqn |-> mast_root map into fqn |-> mast_node_id map + for (fqn, mast_root) in exports.into_iter() { + match mast_forest.find_procedure_root(mast_root) { + Some(node_id) => { + fqn_to_export.insert(fqn, Export::Local(node_id)); + } + None => { + fqn_to_export.insert(fqn, Export::External(mast_root)); + } + } + } + + let digest = content_hash(&fqn_to_export, &mast_forest); + Ok(Self { + digest, + exports: fqn_to_export, mast_forest, - exports, }) } } +/// Accessors impl CompiledLibrary { - /// Returns the inner [`MastForest`]. - pub fn mast_forest(&self) -> &MastForest { - &self.mast_forest + /// Returns the [RpoDigest] representing the content hash of this library + pub fn digest(&self) -> &RpoDigest { + &self.digest } /// Returns the fully qualified name of all procedures exported by the library. - pub fn exports(&self) -> &[FullyQualifiedProcedureName] { - &self.exports + pub fn exports(&self) -> impl Iterator { + self.exports.keys() } + /// Returns the inner [`MastForest`]. + pub fn mast_forest(&self) -> &MastForest { + &self.mast_forest + } +} + +/// Conversions +impl CompiledLibrary { /// Returns an iterator over the module infos of the library. - pub fn into_module_infos(self) -> impl Iterator { + pub fn module_infos(&self) -> impl Iterator { let mut modules_by_path: BTreeMap = BTreeMap::new(); - for (proc_index, proc_name) in self.exports.into_iter().enumerate() { + for (proc_name, export) in self.exports.iter() { modules_by_path .entry(proc_name.module.clone()) .and_modify(|compiled_module| { - let proc_node_id = self.mast_forest.procedure_roots()[proc_index]; - let proc_digest = self.mast_forest[proc_node_id].digest(); + let proc_digest = export.digest(&self.mast_forest); compiled_module.add_procedure_info(ProcedureInfo { name: proc_name.name.clone(), @@ -80,14 +138,13 @@ impl CompiledLibrary { }) }) .or_insert_with(|| { - let proc_node_id = self.mast_forest.procedure_roots()[proc_index]; - let proc_digest = self.mast_forest[proc_node_id].digest(); + let proc_digest = export.digest(&self.mast_forest); let proc = ProcedureInfo { - name: proc_name.name, + name: proc_name.name.clone(), digest: proc_digest, }; - ModuleInfo::new(proc_name.module, vec![proc]) + ModuleInfo::new(proc_name.module.clone(), vec![proc]) }); } @@ -95,6 +152,288 @@ impl CompiledLibrary { } } +/// Serialization +impl CompiledLibrary { + /// Serialize to `target` using `options` + pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { + let Self { + digest: _, + exports, + mast_forest, + } = self; + + options.write_into(target); + mast_forest.write_into(target); + + target.write_usize(exports.len()); + for (proc_name, export) in exports { + proc_name.write_into_with_options(target, options); + export.write_into(target); + } + } +} + +impl Serializable for CompiledLibrary { + fn write_into(&self, target: &mut W) { + self.write_into_with_options(target, AstSerdeOptions::default()) + } +} + +impl Deserializable for CompiledLibrary { + fn read_from(source: &mut R) -> Result { + let options = AstSerdeOptions::read_from(source)?; + let mast_forest = MastForest::read_from(source)?; + + let num_exports = source.read_usize()?; + let mut exports = BTreeMap::new(); + for _ in 0..num_exports { + let proc_name = QualifiedProcedureName::read_from_with_options(source, options)?; + let export = Export::read_with_forest(source, &mast_forest)?; + + exports.insert(proc_name, export); + } + + let digest = content_hash(&exports, &mast_forest); + + Ok(Self { + digest, + exports, + mast_forest, + }) + } +} + +fn content_hash( + exports: &BTreeMap, + mast_forest: &MastForest, +) -> RpoDigest { + let digests = BTreeSet::from_iter(exports.values().map(|export| export.digest(mast_forest))); + digests + .into_iter() + .reduce(|a, b| vm_core::crypto::hash::Rpo256::merge(&[a, b])) + .unwrap() +} + +#[cfg(feature = "std")] +mod use_std_library { + use super::*; + use crate::{ + ast::{self, ModuleKind}, + diagnostics::IntoDiagnostic, + Assembler, + }; + use alloc::collections::btree_map::Entry; + use masl::{LibraryEntry, WalkLibrary}; + use miette::{Context, Report}; + use std::{fs, io, path::Path}; + use vm_core::utils::ReadAdapter; + + impl CompiledLibrary { + /// File extension for the Assembly Library. + pub const LIBRARY_EXTENSION: &'static str = "masl"; + + /// File extension for the Assembly Module. + pub const MODULE_EXTENSION: &'static str = "masm"; + + /// Name of the root module. + pub const MOD: &'static str = "mod"; + + /// Write the library to a target file + /// + /// NOTE: It is up to the caller to use the correct file extension, but there is no + /// specific requirement that the extension be set, or the same as + /// [`Self::LIBRARY_EXTENSION`]. + pub fn write_to_file( + &self, + path: impl AsRef, + options: AstSerdeOptions, + ) -> io::Result<()> { + let path = path.as_ref(); + + if let Some(dir) = path.parent() { + fs::create_dir_all(dir)?; + } + + // NOTE: We catch panics due to i/o errors here due to the fact + // that the ByteWriter trait does not provide fallible APIs, so + // WriteAdapter will panic if the underlying writes fail. This + // needs to be addressed in winterfall at some point + std::panic::catch_unwind(|| { + let mut file = fs::File::create(path)?; + self.write_into_with_options(&mut file, options); + Ok(()) + }) + .map_err(|p| { + match p.downcast::() { + // SAFETY: It is guaranteed safe to read Box + Ok(err) => unsafe { core::ptr::read(&*err) }, + Err(err) => std::panic::resume_unwind(err), + } + })? + } + + /// Create a [CompiledLibrary] from a standard Miden Assembly project layout. + /// + /// The standard layout dictates that a given path is the root of a namespace, and the + /// directory hierarchy corresponds to the namespace hierarchy. A `.masm` file found in a + /// given subdirectory of the root, will be parsed with its [LibraryPath] set based on + /// where it resides in the directory structure. + /// + /// This function recursively parses the entire directory structure under `path`, ignoring + /// any files which do not have the `.masm` extension. + /// + /// For example, let's say I call this function like so: + /// + /// ```rust + /// CompiledLibrary::from_dir("~/masm/std", LibraryNamespace::new("std").unwrap()): + /// ``` + /// + /// Here's how we would handle various files under this path: + /// + /// - ~/masm/std/sys.masm -> Parsed as "std::sys" + /// - ~/masm/std/crypto/hash.masm -> Parsed as "std::crypto::hash" + /// - ~/masm/std/math/u32.masm -> Parsed as "std::math::u32" + /// - ~/masm/std/math/u64.masm -> Parsed as "std::math::u64" + /// - ~/masm/std/math/README.md -> Ignored + pub fn from_dir( + path: impl AsRef, + namespace: LibraryNamespace, + ) -> Result { + let path = path.as_ref(); + if !path.is_dir() { + return Err(Report::msg(format!( + "the provided path '{}' is not a valid directory", + path.display() + ))); + } + + // mod.masm is not allowed in the root directory + if path.join("mod.masm").exists() { + return Err(Report::msg("mod.masm is not allowed in the root directory")); + } + + Self::compile_modules_from_dir(namespace, path) + } + + /// Read the contents (modules) of this library from `dir`, returning any errors that occur + /// while traversing the file system. + /// + /// Errors may also be returned if traversal discovers issues with the library, such as + /// invalid names, etc. + /// + /// Returns a library built from the set of modules that were compiled. + fn compile_modules_from_dir( + namespace: LibraryNamespace, + dir: &Path, + ) -> Result { + let mut modules = BTreeMap::default(); + + let walker = WalkLibrary::new(namespace.clone(), dir) + .into_diagnostic() + .wrap_err_with(|| format!("failed to load library from '{}'", dir.display()))?; + for entry in walker { + let LibraryEntry { + mut name, + source_path, + } = entry?; + if name.last() == Self::MOD { + name.pop(); + } + // Parse module at the given path + let ast = ast::Module::parse_file(name.clone(), ModuleKind::Library, &source_path)?; + match modules.entry(name) { + Entry::Occupied(ref entry) => { + return Err(LibraryError::DuplicateModulePath(entry.key().clone())) + .into_diagnostic(); + } + Entry::Vacant(entry) => { + entry.insert(ast); + } + } + } + + if modules.is_empty() { + return Err(LibraryError::Empty(namespace.clone()).into()); + } + if modules.len() > MAX_MODULES { + return Err(LibraryError::TooManyModulesInLibrary { + name: namespace.clone(), + count: modules.len(), + max: MAX_MODULES, + } + .into()); + } + + Assembler::default().assemble_library(modules.into_values()) + } + + pub fn deserialize_from_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let mut file = fs::File::open(path).map_err(|err| { + DeserializationError::InvalidValue(format!( + "failed to open file at {}: {err}", + path.to_string_lossy() + )) + })?; + let mut adapter = ReadAdapter::new(&mut file); + + Self::read_from(&mut adapter) + } + } +} + +impl Export { + pub fn digest(&self, mast_forest: &MastForest) -> RpoDigest { + match self { + Self::Local(node_id) => mast_forest[*node_id].digest(), + Self::External(digest) => *digest, + } + } + + fn tag(&self) -> u8 { + // SAFETY: This is safe because we have given this enum a primitive representation with + // #[repr(u8)], with the first field of the underlying union-of-structs the discriminant. + // + // See the section on "accessing the numeric value of the discriminant" + // here: https://doc.rust-lang.org/std/mem/fn.discriminant.html + unsafe { *<*const _>::from(self).cast::() } + } +} + +impl Serializable for Export { + fn write_into(&self, target: &mut W) { + target.write_u8(self.tag()); + match self { + Self::Local(node_id) => target.write_u32(node_id.into()), + Self::External(digest) => digest.write_into(target), + } + } +} + +impl Export { + pub fn read_with_forest( + source: &mut R, + mast_forest: &MastForest, + ) -> Result { + match source.read_u8()? { + 0 => { + let node_id = MastNodeId::from_u32_safe(source.read_u32()?, mast_forest)?; + if !mast_forest.is_procedure_root(node_id) { + return Err(DeserializationError::InvalidValue(format!( + "node with id {node_id} is not a procedure root" + ))); + } + Ok(Self::Local(node_id)) + } + 1 => RpoDigest::read_from(source).map(Self::External), + n => Err(DeserializationError::InvalidValue(format!( + "{} is not a valid compiled library export entry", + n + ))), + } + } +} + // KERNEL LIBRARY // ================================================================================================ @@ -110,6 +449,11 @@ pub struct KernelLibrary { } impl KernelLibrary { + /// Returns the inner [`MastForest`]. + pub fn mast_forest(&self) -> &MastForest { + self.library.mast_forest() + } + /// Destructures this kernel library into individual parts. pub fn into_parts(self) -> (Kernel, ModuleInfo, MastForest) { (self.kernel, self.kernel_info, self.library.mast_forest) @@ -124,17 +468,15 @@ impl TryFrom for KernelLibrary { let mut kernel_procs = Vec::with_capacity(library.exports.len()); let mut proc_digests = Vec::with_capacity(library.exports.len()); - for (proc_index, proc_path) in library.exports.iter().enumerate() { - // make sure all procedures are exported directly from the #sys module + for (proc_path, export) in library.exports.iter() { + // make sure all procedures are exported only from the kernel root if proc_path.module != kernel_path { return Err(CompiledLibraryError::InvalidKernelExport { procedure_path: proc_path.clone(), }); } - let proc_node_id = library.mast_forest.procedure_roots()[proc_index]; - let proc_digest = library.mast_forest[proc_node_id].digest(); - + let proc_digest = export.digest(&library.mast_forest); proc_digests.push(proc_digest); kernel_procs.push(ProcedureInfo { name: proc_path.name.clone(), @@ -153,6 +495,66 @@ impl TryFrom for KernelLibrary { } } +/// Serialization +impl KernelLibrary { + /// Serialize to `target` using `options` + pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { + let Self { + kernel: _, + kernel_info: _, + library, + } = self; + + library.write_into_with_options(target, options); + } +} + +impl Serializable for KernelLibrary { + fn write_into(&self, target: &mut W) { + self.write_into_with_options(target, AstSerdeOptions::default()) + } +} + +impl Deserializable for KernelLibrary { + fn read_from(source: &mut R) -> Result { + let library = CompiledLibrary::read_from(source)?; + + Self::try_from(library).map_err(|err| { + DeserializationError::InvalidValue(format!( + "Failed to deserialize kernel library: {err}" + )) + }) + } +} + +#[cfg(feature = "std")] +mod use_std_kernel { + use super::*; + use miette::Report; + use std::{io, path::Path}; + + impl KernelLibrary { + /// Write the library to a target file + pub fn write_to_file( + &self, + path: impl AsRef, + options: AstSerdeOptions, + ) -> io::Result<()> { + self.library.write_to_file(path, options) + } + /// Read a directory and recursively create modules from its `masm` files. + /// + /// For every directory, concatenate the module path with the dir name and proceed. + /// + /// For every file, pick and compile the ones with `masm` extension; skip otherwise. + pub fn from_dir(path: impl AsRef) -> Result { + let library = CompiledLibrary::from_dir(path, LibraryNamespace::Kernel)?; + + Ok(Self::try_from(library)?) + } + } +} + // MODULE INFO // ================================================================================================ @@ -227,56 +629,6 @@ pub struct ProcedureInfo { pub digest: RpoDigest, } -// LIBRARY -// =============================================================================================== - /// Maximum number of modules in a library. +#[cfg(feature = "std")] const MAX_MODULES: usize = u16::MAX as usize; - -/// Maximum number of dependencies in a library. -const MAX_DEPENDENCIES: usize = u16::MAX as usize; - -/// A library definition that provides AST modules for the compilation process. -pub trait Library { - /// Returns the root namespace of this library. - fn root_ns(&self) -> &LibraryNamespace; - - /// Returns the version number of this library. - fn version(&self) -> &Version; - - /// Iterate the modules available in the library. - fn modules(&self) -> impl ExactSizeIterator + '_; - - /// Returns the dependency libraries of this library. - fn dependencies(&self) -> &[LibraryNamespace]; - - /// Returns the module stored at the provided path. - fn get_module(&self, path: &LibraryPath) -> Option<&ast::Module> { - self.modules().find(|&module| module.path() == path) - } -} - -impl Library for &T -where - T: Library, -{ - fn root_ns(&self) -> &LibraryNamespace { - T::root_ns(self) - } - - fn version(&self) -> &Version { - T::version(self) - } - - fn modules(&self) -> impl ExactSizeIterator + '_ { - T::modules(self) - } - - fn dependencies(&self) -> &[LibraryNamespace] { - T::dependencies(self) - } - - fn get_module(&self, path: &LibraryPath) -> Option<&ast::Module> { - T::get_module(self, path) - } -} diff --git a/assembly/src/library/namespace.rs b/assembly/src/library/namespace.rs index 07c1d0d2e3..52ef0a75b0 100644 --- a/assembly/src/library/namespace.rs +++ b/assembly/src/library/namespace.rs @@ -64,6 +64,18 @@ impl LibraryNamespace { source.as_ref().parse() } + /// Construct a new [LibraryNamespace] from a previously-validated [Ident]. + /// + /// NOTE: The caller must ensure that the given identifier is a valid namespace name. + pub fn from_ident_unchecked(name: Ident) -> Self { + match name.as_str() { + Self::KERNEL_PATH => Self::Kernel, + Self::EXEC_PATH => Self::Exec, + Self::ANON_PATH => Self::Anon, + _ => Self::User(name.into_inner()), + } + } + /// Parse a [LibraryNamespace] by taking the prefix of the given path string, and returning /// the namespace and remaining string if successful. pub fn strip_path_prefix(path: &str) -> Result<(Self, &str), LibraryNamespaceError> { diff --git a/assembly/src/library/path.rs b/assembly/src/library/path.rs index 814d7cf54f..5905d69eae 100644 --- a/assembly/src/library/path.rs +++ b/assembly/src/library/path.rs @@ -43,6 +43,26 @@ pub enum LibraryPathComponent<'a> { Normal(&'a Ident), } +impl<'a> LibraryPathComponent<'a> { + /// Get this component as a [prim@str] + #[inline(always)] + pub fn as_str(&self) -> &'a str { + match self { + Self::Namespace(ns) => ns.as_str(), + Self::Normal(id) => id.as_str(), + } + } + + /// Get this component as an [Ident] + #[inline] + pub fn to_ident(&self) -> Ident { + match self { + Self::Namespace(ns) => ns.to_ident(), + Self::Normal(id) => Ident::clone(id), + } + } +} + impl<'a> Eq for LibraryPathComponent<'a> {} impl<'a> PartialEq for LibraryPathComponent<'a> { @@ -76,6 +96,13 @@ impl<'a> fmt::Display for LibraryPathComponent<'a> { } } +impl From> for Ident { + #[inline] + fn from(component: LibraryPathComponent<'_>) -> Self { + component.to_ident() + } +} + /// This is a convenience type alias for a smallvec of [Ident] type Components = smallvec::SmallVec<[Ident; 1]>; @@ -182,13 +209,18 @@ impl LibraryPath { &self.inner.ns } - /// Returns the last component of the path. + /// Returns the last component of the path as a `str` pub fn last(&self) -> &str { + self.last_component().as_str() + } + + /// Returns the last component of the path. + pub fn last_component(&self) -> LibraryPathComponent<'_> { self.inner .components .last() - .map(|component| component.as_str()) - .unwrap_or_else(|| self.inner.ns.as_str()) + .map(LibraryPathComponent::Normal) + .unwrap_or_else(|| LibraryPathComponent::Namespace(&self.inner.ns)) } /// Returns the number of components in the path. diff --git a/assembly/src/library/tests.rs b/assembly/src/library/tests.rs index 476f96f6c6..103ad0aadf 100644 --- a/assembly/src/library/tests.rs +++ b/assembly/src/library/tests.rs @@ -1,12 +1,14 @@ use alloc::{string::ToString, sync::Arc, vec::Vec}; -use vm_core::utils::{Deserializable, Serializable, SliceReader}; +use vm_core::utils::SliceReader; -use super::{Library, LibraryNamespace, LibraryPath, MaslLibrary, Version}; +use super::LibraryPath; use crate::{ - ast::{AstSerdeOptions, Module, ModuleKind}, + ast::{AstSerdeOptions, Module, ModuleKind, ProcedureName}, diagnostics::{IntoDiagnostic, Report, SourceFile}, + library::CompiledLibrary, testing::TestContext, + Assembler, Deserializable, }; macro_rules! parse_module { @@ -41,26 +43,23 @@ fn masl_locations_serialization() -> Result<(), Report> { end "#; let bar = parse_module!("test::bar", bar); - let modules = vec![foo, bar]; + let modules = [foo, bar]; // serialize/deserialize the bundle with locations - let namespace = LibraryNamespace::new("test").unwrap(); - let version = Version::min(); - let bundle = MaslLibrary::new(namespace, version, modules.iter().cloned(), Vec::new())?; + let bundle = Assembler::default().assemble_library(modules.iter().cloned()).unwrap(); let mut bytes = Vec::new(); - bundle.write_into(&mut bytes); - let deserialized = MaslLibrary::read_from(&mut SliceReader::new(&bytes)).unwrap(); + bundle.write_into_with_options(&mut bytes, AstSerdeOptions::new(true, true)); + let deserialized = CompiledLibrary::read_from(&mut SliceReader::new(&bytes)).unwrap(); assert_eq!(bundle, deserialized); // serialize/deserialize the bundle without locations - let namespace = LibraryNamespace::new("test").unwrap(); - let bundle = MaslLibrary::new(namespace, version, modules, Vec::new())?; + let bundle = Assembler::default().assemble_library(modules.iter().cloned()).unwrap(); // serialize/deserialize the bundle let mut bytes = Vec::new(); bundle.write_into_with_options(&mut bytes, AstSerdeOptions::new(true, false)); - let deserialized = MaslLibrary::read_from(&mut SliceReader::new(&bytes)).unwrap(); + let deserialized = CompiledLibrary::read_from(&mut SliceReader::new(&bytes)).unwrap(); assert_eq!(bundle, deserialized); Ok(()) @@ -76,23 +75,16 @@ fn get_module_by_path() -> Result<(), Report> { end "#; let foo = parse_module!("test::foo", foo_source); - let modules = vec![foo]; + let modules = [foo]; // create the bundle with locations - let namespace = LibraryNamespace::new("test")?; - let version = Version::min(); - let bundle = MaslLibrary::new(namespace, version, modules, Vec::new())?; + let bundle = Assembler::default().assemble_library(modules.iter().cloned()).unwrap(); - // get AST associated with "test::foo" path - let foo_ast = bundle.get_module(&LibraryPath::new("test::foo").unwrap()).unwrap(); - let foo_expected = "export.foo - add -end + let foo_module_info = bundle.module_infos().next().unwrap(); + assert_eq!(foo_module_info.path(), &LibraryPath::new("test::foo").unwrap()); -"; - assert_eq!(foo_ast.to_string(), foo_expected); - - assert!(bundle.get_module(&LibraryPath::new("test::bar").unwrap()).is_none()); + let (_, foo_proc) = foo_module_info.procedure_infos().next().unwrap(); + assert_eq!(foo_proc.name, ProcedureName::new("foo").unwrap()); Ok(()) } diff --git a/assembly/src/parser/grammar.lalrpop b/assembly/src/parser/grammar.lalrpop index 77b2aefd8f..cd5861d4ae 100644 --- a/assembly/src/parser/grammar.lalrpop +++ b/assembly/src/parser/grammar.lalrpop @@ -314,7 +314,7 @@ Proc: Form = { error: ParsingError::UnnamedReexportOfMastRoot { span }, }); } - ProcedureAlias::new(alias.unwrap(), digest, true) + ProcedureAlias::new(alias.unwrap(), AliasTarget::MastRoot(digest)) } InvocationTarget::ProcedurePath { name, module } => { let export_name = alias.unwrap_or_else(|| name.clone()); @@ -324,21 +324,21 @@ Proc: Form = { LibraryNamespace::ANON_PATH => LibraryPath::new_from_components(LibraryNamespace::Anon, []), _ => LibraryPath::new_from_components(LibraryNamespace::User(module.into_inner()), []), }; - let target = FullyQualifiedProcedureName { + let target = QualifiedProcedureName { span, module, name, }; - ProcedureAlias::new(export_name, target, false) + ProcedureAlias::new(export_name, AliasTarget::ProcedurePath(target)) } InvocationTarget::AbsoluteProcedurePath { name, path } => { let export_name = alias.unwrap_or_else(|| name.clone()); - let target = FullyQualifiedProcedureName { + let target = QualifiedProcedureName { span, module: path, name, }; - ProcedureAlias::new(export_name, target, true) + ProcedureAlias::new(export_name, AliasTarget::AbsoluteProcedurePath(target)) } }; let alias = alias.with_source_file(Some(source_file.clone())); diff --git a/assembly/src/sema/mod.rs b/assembly/src/sema/mod.rs index 3df61e5afe..35c1d3d613 100644 --- a/assembly/src/sema/mod.rs +++ b/assembly/src/sema/mod.rs @@ -7,7 +7,7 @@ pub use self::errors::{SemanticAnalysisError, SyntaxError}; use self::passes::{ConstEvalVisitor, VerifyInvokeTargets}; -use crate::{ast::*, diagnostics::SourceFile, LibraryNamespace, LibraryPath, Span, Spanned}; +use crate::{ast::*, diagnostics::SourceFile, LibraryPath, Spanned}; use alloc::collections::BTreeSet; use alloc::{boxed::Box, collections::VecDeque, sync::Arc, vec::Vec}; @@ -179,16 +179,11 @@ fn visit_procedures( // to its fully-qualified path. This is needed because after // parsing, the path only contains the last component, // e.g. `u64` of `std::math::u64`. - let is_absolute = alias.absolute(); - let target = &mut alias.target; + let is_absolute = alias.is_absolute(); if !is_absolute { - if let AliasTarget::Path(ref mut target) = target { - let imported_module = match target.module.namespace() { - LibraryNamespace::User(ref ns) => { - Ident::new_unchecked(Span::new(target.span(), ns.clone())) - } - _ => unreachable!(), - }; + if let AliasTarget::ProcedurePath(ref mut target) = alias.target_mut() { + let imported_module = + target.module.namespace().to_ident().with_span(target.span); if let Some(import) = module.resolve_import_mut(&imported_module) { target.module = import.path.clone(); // Mark the backing import as used diff --git a/assembly/src/testing.rs b/assembly/src/testing.rs index b4f512fa85..95faeb56c7 100644 --- a/assembly/src/testing.rs +++ b/assembly/src/testing.rs @@ -5,7 +5,8 @@ use crate::{ reporting::{set_hook, ReportHandlerOpts}, Report, SourceFile, }, - Compile, CompileOptions, Library, LibraryPath, RpoDigest, + library::CompiledLibrary, + Compile, CompileOptions, LibraryPath, RpoDigest, }; #[cfg(feature = "std")] @@ -295,11 +296,8 @@ impl TestContext { /// Add the modules of `library` to the [Assembler] constructed by this context. #[track_caller] - pub fn add_library(&mut self, library: &L) -> Result<(), Report> - where - L: ?Sized + Library + 'static, - { - self.assembler.add_library(library) + pub fn add_library(&mut self, library: impl AsRef) -> Result<(), Report> { + self.assembler.add_compiled_library(library) } /// Compile a [Program] from `source` using the [Assembler] constructed by this context. diff --git a/assembly/src/tests.rs b/assembly/src/tests.rs index 02734bc1f7..57af5bd3da 100644 --- a/assembly/src/tests.rs +++ b/assembly/src/tests.rs @@ -1,4 +1,4 @@ -use alloc::{rc::Rc, string::ToString, vec::Vec}; +use alloc::string::ToString; use crate::{ assert_diagnostic_lines, @@ -6,7 +6,7 @@ use crate::{ diagnostics::Report, regex, source_file, testing::{Pattern, TestContext}, - Assembler, Library, LibraryNamespace, LibraryPath, MaslLibrary, ModuleParser, Version, + Assembler, LibraryPath, ModuleParser, }; type TestResult = Result<(), Report>; @@ -411,25 +411,12 @@ fn get_proc_name_of_unknown_module() -> TestResult { let module_path_one = "module::path::one".parse().unwrap(); let module1 = context.parse_module_with_path(module_path_one, module_source1)?; - let masl_lib = - MaslLibrary::new(module1.namespace().clone(), Version::default(), [module1], vec![]) - .unwrap(); - - // instantiate assembler - context.add_library(&masl_lib)?; - - // compile program with procref calls - let source = source_file!( - " - use.module::path::one + let report = Assembler::default() + .assemble_library(core::iter::once(module1)) + .expect_err("expected unknown module error"); - begin - procref.one::foo - end" - ); - assert_assembler_diagnostic!( - context, - source, + assert_diagnostic_lines!( + report, "undefined module 'module::path::two'", regex!(r#",-\[test[\d]+:5:22\]"#), "4 | export.foo", @@ -438,6 +425,7 @@ fn get_proc_name_of_unknown_module() -> TestResult { "6 | end", " `----" ); + Ok(()) } @@ -1518,8 +1506,7 @@ fn program_with_one_import_and_hex_call() -> TestResult { let mut context = TestContext::default(); let path = MODULE.parse().unwrap(); let ast = context.parse_module_with_path(path, source_file!(PROCEDURE.to_string()))?; - let ns = ast.namespace().clone(); - let library = DummyLibrary::new(ns, vec![Rc::from(ast)]); + let library = Assembler::default().assemble_library(core::iter::once(ast)).unwrap(); context.add_library(&library)?; @@ -1539,28 +1526,7 @@ begin join join basic_block push(4) push(3) end - join - join - join - basic_block eqz end - basic_block swap eqz and end - end - join - basic_block swap eqz and end - basic_block swap eqz and end - end - end - join - join - basic_block swap eqz and end - basic_block swap eqz and end - end - join - basic_block swap eqz and end - basic_block swap eqz and end - end - end - end + external.0x20234ee941e53a15886e733cc8e041198c6e90d2a16ea18ce1030e8c3596dd38 end call.0x20234ee941e53a15886e733cc8e041198c6e90d2a16ea18ce1030e8c3596dd38 end @@ -1594,8 +1560,7 @@ fn program_with_two_imported_procs_with_same_mast_root() -> TestResult { let mut context = TestContext::default(); let path = MODULE.parse().unwrap(); let ast = context.parse_module_with_path(path, source_file!(PROCEDURE.to_string()))?; - let ns = ast.namespace().clone(); - let library = DummyLibrary::new(ns, vec![Rc::from(ast)]); + let library = Assembler::default().assemble_library(core::iter::once(ast)).unwrap(); context.add_library(&library)?; @@ -1664,8 +1629,7 @@ fn program_with_reexported_proc_in_same_library() -> TestResult { Module::parse_str(REF_MODULE.parse().unwrap(), ModuleKind::Library, REF_MODULE_BODY) .unwrap(); - let ns = ref_ast.namespace().clone(); - let library = DummyLibrary::new(ns, vec![Rc::from(ast), Rc::from(ref_ast)]); + let library = Assembler::default().assemble_library([ast, ref_ast]).unwrap(); context.add_library(&library)?; @@ -1684,9 +1648,9 @@ begin join join basic_block push(4) push(3) end - basic_block u32assert2(0) eqz swap eqz and end + external.0xb9691da1d9b4b364aca0a0990e9f04c446a2faa622c8dd0d8831527dbec61393 end - basic_block eqz swap eqz and end + external.0xcb08c107c81c582788cbf63c99f6b455e11b33bb98ca05fe1cfa17c087dfa8f1 end end"; assert_str_eq!(format!("{program}"), expected); @@ -1721,18 +1685,16 @@ fn program_with_reexported_proc_in_another_library() -> TestResult { "#; let mut context = TestContext::default(); - let ast = Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, MODULE_BODY).unwrap(); - let ns = ast.namespace().clone(); - let dummy_library_1 = DummyLibrary::new(ns, vec![Rc::from(ast)]); - + // We reference code in this module let ref_ast = - Module::parse_str(REF_MODULE.parse().unwrap(), ModuleKind::Library, REF_MODULE_BODY) - .unwrap(); - let ns = ref_ast.namespace().clone(); - let dummy_library_2 = DummyLibrary::new(ns, vec![Rc::from(ref_ast)]); + Module::parse_str(REF_MODULE.parse().unwrap(), ModuleKind::Library, REF_MODULE_BODY)?; + // But only exports from this module are exposed by the library + let ast = Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, MODULE_BODY)?; + + let dummy_library = Assembler::default().with_module(ref_ast)?.assemble_library([ast])?; - context.add_library(&dummy_library_1)?; - context.add_library(&dummy_library_2)?; + // Now we want to use the the library we've compiled + context.add_library(&dummy_library)?; let source = source_file!(format!( r#" @@ -1750,24 +1712,24 @@ begin join join basic_block push(4) push(3) end - basic_block u32assert2(0) eqz swap eqz and end + external.0xb9691da1d9b4b364aca0a0990e9f04c446a2faa622c8dd0d8831527dbec61393 end - basic_block eqz swap eqz and end + external.0xcb08c107c81c582788cbf63c99f6b455e11b33bb98ca05fe1cfa17c087dfa8f1 end end"; assert_str_eq!(format!("{program}"), expected); - // when the re-exported proc is part of a different library and the library is not passed to - // the assembler it should fail + // We also want to assert that exports from the referenced module do not leak let mut context = TestContext::default(); - context.add_library(&dummy_library_1)?; + context.add_library(dummy_library)?; + let source = source_file!(format!( r#" - use.{MODULE} + use.{REF_MODULE} begin push.4 push.3 - exec.u256::checked_eqz - exec.u256::notchecked_eqz + exec.u64::checked_eqz + exec.u64::notchecked_eqz end"# )); assert_assembler_diagnostic!(context, source, "undefined module 'dummy2::math::u64'"); @@ -1793,8 +1755,7 @@ fn module_alias() -> TestResult { let mut context = TestContext::default(); let ast = Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, PROCEDURE).unwrap(); - let ns = ast.namespace().clone(); - let library = DummyLibrary::new(ns, vec![Rc::from(ast)]); + let library = Assembler::default().assemble_library([ast]).unwrap(); context.add_library(&library)?; @@ -1814,18 +1775,7 @@ fn module_alias() -> TestResult { begin join basic_block pad incr pad push(2) pad end - basic_block - swap - movup3 - u32assert2(0) - u32add - movup3 - movup3 - u32assert2(0) - u32add3 - eqz - assert(0) - end + external.0x3cff5b58a573dc9d25fd3c57130cc57e5b1b381dc58b5ae3594b390c59835e63 end end"; assert_str_eq!(format!("{program}"), expected); @@ -2393,7 +2343,7 @@ fn test_compiled_library() { export.foo push.7 add.5 - end + end # Same definition as mod1::foo export.bar push.1 @@ -2406,15 +2356,15 @@ fn test_compiled_library() { let compiled_library = { let assembler = Assembler::default(); - assembler.assemble_library(vec![mod1, mod2].into_iter()).unwrap() + assembler.assemble_library([mod1, mod2]).unwrap() }; - assert_eq!(compiled_library.exports().len(), 4); + assert_eq!(compiled_library.exports().count(), 4); // Compile program that uses compiled library let mut assembler = Assembler::default(); - assembler.add_compiled_library(compiled_library).unwrap(); + assembler.add_compiled_library(&compiled_library).unwrap(); let program_source = " use.mylib::mod1 @@ -2436,41 +2386,3 @@ fn test_compiled_library() { let _program = assembler.assemble_program(program_source).unwrap(); } - -// DUMMY LIBRARY -// ================================================================================================ - -struct DummyLibrary { - namespace: LibraryNamespace, - modules: Vec>, - dependencies: Vec, -} - -impl DummyLibrary { - fn new(namespace: LibraryNamespace, modules: Vec>) -> Self { - Self { - namespace, - modules, - dependencies: Vec::new(), - } - } -} - -impl Library for DummyLibrary { - fn root_ns(&self) -> &LibraryNamespace { - &self.namespace - } - - fn version(&self) -> &Version { - const MIN: Version = Version::min(); - &MIN - } - - fn modules(&self) -> impl ExactSizeIterator + '_ { - self.modules.iter().map(|p| p.as_ref()) - } - - fn dependencies(&self) -> &[LibraryNamespace] { - &self.dependencies - } -} diff --git a/core/src/lib.rs b/core/src/lib.rs index fa0ad48523..7be244471b 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -97,6 +97,21 @@ pub use math::{ pub mod prettier { pub use miden_formatting::{prettier::*, pretty_via_display, pretty_via_to_string}; + + /// Pretty-print a list of [PrettyPrint] values as comma-separated items. + pub fn pretty_print_csv<'a, T>(items: impl IntoIterator) -> Document + where + T: PrettyPrint + 'a, + { + let mut doc = Document::Empty; + for (i, item) in items.into_iter().enumerate() { + if i > 0 { + doc += const_text(", "); + } + doc += item.render(); + } + doc + } } mod operations; diff --git a/core/src/mast/mod.rs b/core/src/mast/mod.rs index 8f3483aad0..42a1272360 100644 --- a/core/src/mast/mod.rs +++ b/core/src/mast/mod.rs @@ -158,6 +158,11 @@ impl MastForest { self.roots.iter().find(|&&root_id| self[root_id].digest() == digest).copied() } + /// Returns true if a node with the specified ID is a root of a procedure in this MAST forest. + pub fn is_procedure_root(&self, node_id: MastNodeId) -> bool { + self.roots.contains(&node_id) + } + /// Returns an iterator over the digest of the procedures in this MAST forest. pub fn procedure_digests(&self) -> impl Iterator + '_ { self.roots.iter().map(|&root_id| self[root_id].digest()) diff --git a/miden/benches/program_compilation.rs b/miden/benches/program_compilation.rs index 4462b7231d..417e506918 100644 --- a/miden/benches/program_compilation.rs +++ b/miden/benches/program_compilation.rs @@ -15,9 +15,11 @@ fn program_compilation(c: &mut Criterion) { exec.sha256::hash_2to1 end"; bench.iter(|| { - let assembler = Assembler::default() - .with_library(&StdLibrary::default()) + let mut assembler = Assembler::default(); + assembler + .add_compiled_library(StdLibrary::default()) .expect("failed to load stdlib"); + assembler.assemble_program(source).expect("Failed to compile test source.") }); }); diff --git a/miden/benches/program_execution.rs b/miden/benches/program_execution.rs index 4487672ca9..6b7556f15e 100644 --- a/miden/benches/program_execution.rs +++ b/miden/benches/program_execution.rs @@ -15,8 +15,9 @@ fn program_execution(c: &mut Criterion) { begin exec.sha256::hash_2to1 end"; - let assembler = Assembler::default() - .with_library(&StdLibrary::default()) + let mut assembler = Assembler::default(); + assembler + .add_compiled_library(StdLibrary::default()) .expect("failed to load stdlib"); let program: Program = assembler.assemble_program(source).expect("Failed to compile test source."); diff --git a/miden/src/cli/bundle.rs b/miden/src/cli/bundle.rs index 83d87abaec..d660982d55 100644 --- a/miden/src/cli/bundle.rs +++ b/miden/src/cli/bundle.rs @@ -1,6 +1,8 @@ use assembly::{ + ast::AstSerdeOptions, diagnostics::{IntoDiagnostic, Report}, - LibraryNamespace, MaslLibrary, Version, + library::CompiledLibrary, + LibraryNamespace, Version, }; use clap::Parser; use std::path::PathBuf; @@ -40,11 +42,17 @@ impl BundleCmd { let library_namespace = namespace.parse::().expect("invalid base namespace"); - let version = self.version.parse::().expect("invalid cargo version"); - let stdlib = MaslLibrary::read_from_dir(&self.dir, library_namespace, version)?; + // TODO: Add version to `Library` + let _version = self.version.parse::().expect("invalid cargo version"); + let stdlib = CompiledLibrary::from_dir(&self.dir, library_namespace)?; // write the masl output - stdlib.write_to_dir(self.dir.clone()).into_diagnostic()?; + let options = AstSerdeOptions::new(false, false); + let output_file = self + .dir + .join(self.namespace.as_deref().unwrap_or("out")) + .with_extension(CompiledLibrary::LIBRARY_EXTENSION); + stdlib.write_to_file(output_file, options).into_diagnostic()?; println!("Built library {}", namespace); diff --git a/miden/src/cli/data.rs b/miden/src/cli/data.rs index 76355a3601..333f187236 100644 --- a/miden/src/cli/data.rs +++ b/miden/src/cli/data.rs @@ -1,7 +1,8 @@ use assembly::{ ast::{Module, ModuleKind}, diagnostics::{IntoDiagnostic, Report, WrapErr}, - Assembler, Library, LibraryNamespace, MaslLibrary, + library::CompiledLibrary, + Assembler, LibraryNamespace, }; use miden_vm::{ crypto::{MerkleStore, MerkleTree, NodeIndex, PartialMerkleTree, RpoDigest, SimpleSmt}, @@ -404,20 +405,19 @@ impl ProgramFile { /// Compiles this program file into a [Program]. #[instrument(name = "compile_program", skip_all)] - pub fn compile<'a, I, L>(&self, debug: &Debug, libraries: I) -> Result + pub fn compile<'a, I>(&self, debug: &Debug, libraries: I) -> Result where - I: IntoIterator, - L: ?Sized + Library + 'static, + I: IntoIterator, { // compile program - let mut assembler = Assembler::default() - .with_debug_mode(debug.is_on()) - .with_library(&StdLibrary::default()) + let mut assembler = Assembler::default().with_debug_mode(debug.is_on()); + assembler + .add_compiled_library(StdLibrary::default()) .wrap_err("Failed to load stdlib")?; - assembler = assembler - .with_libraries(libraries.into_iter()) - .wrap_err("Failed to load libraries")?; + for library in libraries { + assembler.add_compiled_library(library).wrap_err("Failed to load libraries")?; + } let program: Program = assembler .assemble_program(self.ast.as_ref()) @@ -529,7 +529,7 @@ impl ProgramHash { // LIBRARY FILE // ================================================================================================ pub struct Libraries { - pub libraries: Vec, + pub libraries: Vec, } impl Libraries { @@ -543,7 +543,9 @@ impl Libraries { let mut libraries = Vec::new(); for path in paths { - let library = MaslLibrary::read_from_file(path)?; + // TODO(plafer): How to create a `Report` from an error that doesn't derive + // `Diagnostic`? + let library = CompiledLibrary::deserialize_from_file(path).unwrap(); libraries.push(library); } diff --git a/miden/src/examples/blake3.rs b/miden/src/examples/blake3.rs index 7bae853923..f0411bb785 100644 --- a/miden/src/examples/blake3.rs +++ b/miden/src/examples/blake3.rs @@ -45,7 +45,7 @@ fn generate_blake3_program(n: usize) -> Program { ); Assembler::default() - .with_library(&StdLibrary::default()) + .with_compiled_library(StdLibrary::default()) .unwrap() .assemble_program(program) .unwrap() diff --git a/miden/src/repl/mod.rs b/miden/src/repl/mod.rs index 1139f20e42..b96b220928 100644 --- a/miden/src/repl/mod.rs +++ b/miden/src/repl/mod.rs @@ -1,4 +1,4 @@ -use assembly::{Assembler, Library, MaslLibrary}; +use assembly::{library::CompiledLibrary, Assembler}; use miden_vm::{math::Felt, DefaultHost, StackInputs, Word}; use processor::ContextId; use rustyline::{error::ReadlineError, DefaultEditor}; @@ -151,13 +151,13 @@ pub fn start_repl(library_paths: &Vec, use_stdlib: bool) { // load libraries from files let mut provided_libraries = Vec::new(); for path in library_paths { - let library = MaslLibrary::read_from_file(path) + let library = CompiledLibrary::deserialize_from_file(path) .map_err(|e| format!("Failed to read library: {e}")) .unwrap(); provided_libraries.push(library); } if use_stdlib { - provided_libraries.push(MaslLibrary::from(StdLibrary::default())); + provided_libraries.push(StdLibrary::default().into()); } println!("========================== Miden REPL ============================"); @@ -303,14 +303,16 @@ pub fn start_repl(library_paths: &Vec, use_stdlib: bool) { #[allow(clippy::type_complexity)] fn execute( program: String, - provided_libraries: &[MaslLibrary], + provided_libraries: &[CompiledLibrary], ) -> Result<(Vec<(u64, Word)>, Vec), String> { // compile program let mut assembler = Assembler::default(); - assembler = assembler - .with_libraries(provided_libraries.iter()) - .map_err(|err| format!("{err}"))?; + for library in provided_libraries { + assembler + .add_compiled_library(library.clone()) + .map_err(|err| format!("{err}"))?; + } let program = assembler.assemble_program(program).map_err(|err| format!("{err}"))?; @@ -357,7 +359,7 @@ fn read_mem_address(mem_str: &str) -> Result { /// all available modules if no module name was provided. fn handle_use_command( line: String, - provided_libraries: &Vec, + provided_libraries: &[CompiledLibrary], imported_modules: &mut BTreeSet, ) { let tokens: Vec<&str> = line.split_whitespace().collect(); @@ -366,7 +368,7 @@ fn handle_use_command( 1 => { println!("Modules available for importing:"); for lib in provided_libraries { - lib.modules().for_each(|module| println!("{}", module.path())); + lib.module_infos().for_each(|module| println!("{}", module.path())); } } 2 => { diff --git a/miden/src/tools/mod.rs b/miden/src/tools/mod.rs index f4718ce014..627f6df0e7 100644 --- a/miden/src/tools/mod.rs +++ b/miden/src/tools/mod.rs @@ -216,7 +216,7 @@ where { let program = Assembler::default() .with_debug_mode(true) - .with_library(&StdLibrary::default())? + .with_compiled_library(StdLibrary::default())? .assemble_program(program)?; let mut execution_details = ExecutionDetails::default(); diff --git a/miden/tests/integration/cli/cli_test.rs b/miden/tests/integration/cli/cli_test.rs index 34b42bd7aa..ccadae4d0f 100644 --- a/miden/tests/integration/cli/cli_test.rs +++ b/miden/tests/integration/cli/cli_test.rs @@ -12,7 +12,10 @@ fn cli_run() -> Result<(), Box> { .current_release() .current_target() .run() - .unwrap(); + .unwrap_or_else(|err| { + eprintln!("{err}"); + panic!("failed to build `miden`"); + }); let mut cmd = bin_under_test.command(); diff --git a/miden/tests/integration/flow_control/mod.rs b/miden/tests/integration/flow_control/mod.rs index 9d2954f960..ed34ff9b0a 100644 --- a/miden/tests/integration/flow_control/mod.rs +++ b/miden/tests/integration/flow_control/mod.rs @@ -1,4 +1,6 @@ use assembly::{ast::ModuleKind, Assembler, LibraryPath}; +use core::iter; +use miden_vm::Module; use processor::ExecutionError; use prover::Digest; use stdlib::StdLibrary; @@ -187,8 +189,6 @@ fn local_fn_call_with_mem_access() { test.prove_and_verify(vec![3, 7], false); } -// TODO: Fix test after we implement the new `Assembler::add_library()` -#[ignore] #[test] fn simple_syscall() { let kernel_source = " @@ -204,7 +204,7 @@ fn simple_syscall() { // TODO: update and use macro? let test = Test { - kernel: Some(kernel_source.to_string()), + kernel_source: Some(kernel_source.to_string()), stack_inputs: StackInputs::try_from_ints([1, 2]).unwrap(), ..Test::new(&format!("test{}", line!()), program_source, false) }; @@ -389,13 +389,8 @@ fn simple_dyncall() { // PROCREF INSTRUCTION // ================================================================================================ -// TODO: Fix test after we implement the new `Assembler::add_library()` -#[ignore] -#[allow(unused)] #[test] fn procref() { - let mut assembler = Assembler::default().with_library(&StdLibrary::default()).unwrap(); - let module_source = " use.std::math::u64 export.u64::overflowing_add @@ -406,12 +401,19 @@ fn procref() { "; // obtain procedures' MAST roots by compiling them as module - let module_path = "test::foo".parse::().unwrap(); - let opts = assembly::CompileOptions::new(ModuleKind::Library, module_path).unwrap(); - - // TODO: Fix - // let mast_roots = assembler.assemble_module(module_source, opts).unwrap(); - let mast_roots: Vec = Vec::new(); + let mast_roots: Vec = { + let module_path = "test::foo".parse::().unwrap(); + let module = Module::parse_str(module_path, ModuleKind::Library, module_source).unwrap(); + let library = Assembler::default() + .with_compiled_library(StdLibrary::default()) + .unwrap() + .assemble_library(iter::once(module)) + .unwrap(); + + let module_info = library.module_infos().next().unwrap(); + + module_info.procedure_digests().collect() + }; let source = " use.std::math::u64 @@ -430,15 +432,15 @@ fn procref() { test.libraries = vec![StdLibrary::default().into()]; test.expect_stack(&[ - mast_roots[1][3].as_int(), - mast_roots[1][2].as_int(), - mast_roots[1][1].as_int(), - mast_roots[1][0].as_int(), - 0, mast_roots[0][3].as_int(), mast_roots[0][2].as_int(), mast_roots[0][1].as_int(), mast_roots[0][0].as_int(), + 0, + mast_roots[1][3].as_int(), + mast_roots[1][2].as_int(), + mast_roots[1][1].as_int(), + mast_roots[1][0].as_int(), ]); test.prove_and_verify(vec![], false); diff --git a/miden/tests/integration/operations/io_ops/env_ops.rs b/miden/tests/integration/operations/io_ops/env_ops.rs index 58e2230954..8744d9aafb 100644 --- a/miden/tests/integration/operations/io_ops/env_ops.rs +++ b/miden/tests/integration/operations/io_ops/env_ops.rs @@ -126,8 +126,6 @@ fn locaddr() { // CALLER INSTRUCTION // ================================================================================================ -// TODO: Fix test after we implement the new `Assembler::add_library()` -#[ignore] #[test] fn caller() { let kernel_source = " @@ -147,7 +145,7 @@ fn caller() { // TODO: update and use macro? let test = Test { - kernel: Some(kernel_source.to_string()), + kernel_source: Some(kernel_source.to_string()), stack_inputs: StackInputs::try_from_ints([1, 2, 3, 4, 5]).unwrap(), ..Test::new(&format!("test{}", line!()), program_source, false) }; diff --git a/stdlib/build.rs b/stdlib/build.rs index 7b059b5ff0..42d03bfaa2 100644 --- a/stdlib/build.rs +++ b/stdlib/build.rs @@ -1,18 +1,16 @@ use assembly::{ + ast::AstSerdeOptions, diagnostics::{IntoDiagnostic, Result}, - LibraryNamespace, MaslLibrary, Version, + library::CompiledLibrary, + LibraryNamespace, Version, }; -use std::{env, fs, io, path::Path}; - -mod md_renderer; -use md_renderer::MarkdownRenderer; +use std::{env, path::Path}; // CONSTANTS // ================================================================================================ -const ASM_DIR_PATH: &str = "./asm"; -const ASL_DIR_PATH: &str = "./assets"; -const DOC_DIR_PATH: &str = "./docs"; +const ASM_DIR_PATH: &str = "asm"; +const ASL_DIR_PATH: &str = "assets"; // PRE-PROCESSING // ================================================================================================ @@ -25,51 +23,23 @@ fn main() -> Result<()> { println!("cargo:rerun-if-changed=asm"); println!("cargo:rerun-if-changed=../assembly/src"); + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let asm_dir = Path::new(manifest_dir).join(ASM_DIR_PATH); + let namespace = "std".parse::().expect("invalid base namespace"); - let version = env!("CARGO_PKG_VERSION").parse::().expect("invalid cargo version"); - let stdlib = MaslLibrary::read_from_dir(ASM_DIR_PATH, namespace, version)?; + // TODO: Add version to `Library` + let _version = env!("CARGO_PKG_VERSION").parse::().expect("invalid cargo version"); + let stdlib = CompiledLibrary::from_dir(asm_dir, namespace)?; // write the masl output let build_dir = env::var("OUT_DIR").unwrap(); - stdlib - .write_to_dir(Path::new(&build_dir).join(ASL_DIR_PATH)) - .into_diagnostic()?; - - // updates the documentation of these modules - build_stdlib_docs(&stdlib, DOC_DIR_PATH).into_diagnostic()?; - - Ok(()) -} - -// STDLIB DOCUMENTATION -// ================================================================================================ - -/// A renderer renders a ModuleSourceMap into a particular doc format and index (e.g: markdown, etc) -trait Renderer { - // Render writes out the document files into the output directory - fn render(stdlib: &MaslLibrary, output_dir: &str); -} - -/// Writes Miden standard library modules documentation markdown files based on the available -/// modules and comments. -pub fn build_stdlib_docs(library: &MaslLibrary, output_dir: &str) -> io::Result<()> { - // Clean the output folder. This only deletes the folder's content, and not the folder itself, - // because removing the folder fails on docs.rs - for entry in fs::read_dir(output_dir)? { - let entry = entry?; - let metadata = entry.metadata()?; - - if metadata.is_dir() { - fs::remove_dir_all(entry.path())?; - } else { - assert!(metadata.is_file()); - fs::remove_file(entry.path())?; - } - } - - // Render the stdlib modules into markdown - // TODO: Make the renderer choice pluggable. - MarkdownRenderer::render(library, output_dir); + let build_dir = Path::new(&build_dir); + let options = AstSerdeOptions::new(false, false); + let output_file = build_dir + .join(ASL_DIR_PATH) + .join("std") + .with_extension(CompiledLibrary::LIBRARY_EXTENSION); + stdlib.write_to_file(output_file, options).into_diagnostic()?; Ok(()) } diff --git a/stdlib/md_renderer.rs b/stdlib/md_renderer.rs deleted file mode 100644 index 34953573b8..0000000000 --- a/stdlib/md_renderer.rs +++ /dev/null @@ -1,117 +0,0 @@ -use crate::Renderer; -use assembly::{ - ast::{Export, Module, Procedure, ProcedureAlias}, - Library, LibraryPath, MaslLibrary, -}; -use std::{ - borrow::Cow, - fs::{self, File}, - io::Write, - path::PathBuf, -}; - -// MARKDOWN RENDERER -// ================================================================================================ - -pub struct MarkdownRenderer {} - -impl MarkdownRenderer { - fn escape_module_docs(s: &str) -> Cow<'_, str> { - if s.contains('\n') { - Cow::Owned(s.to_string().replace('\n', "
")) - } else { - Cow::Borrowed(s) - } - } - - fn escape_procedure_docs(s: &str) -> Cow<'_, str> { - if s.contains(['|', '\n']) { - Cow::Owned(s.to_string().replace('|', "\\|").replace('\n', "
")) - } else { - Cow::Borrowed(s) - } - } - - fn write_docs_header(mut writer: &File, module: &Module) { - let ns = module.path(); - let header = - format!("\n## {ns}\n| Procedure | Description |\n| ----------- | ------------- |\n"); - writer.write_all(header.as_bytes()).expect("unable to write header to writer"); - } - - fn write_docs_procedure(mut writer: &File, proc: &Procedure) { - if let Some(docs) = proc.docs() { - let escaped = Self::escape_procedure_docs(docs); - writer - .write_fmt(format_args!("| {} | {} |\n", proc.name(), &escaped,)) - .expect("unable to write procedure to writer"); - } - } - - fn write_docs_reexported_proc(mut writer: &File, proc: &ProcedureAlias) { - if let Some(docs) = proc.docs() { - let escaped = Self::escape_procedure_docs(docs); - writer - .write_fmt(format_args!("| {} | {} |\n", proc.name(), &escaped,)) - .expect("unable to write procedure alias to writer"); - } - } - - fn write_docs_module(mut writer: &File, module: &Module) { - if let Some(docs) = module.docs() { - let escaped = Self::escape_module_docs(docs.into_inner()); - writer.write_all(escaped.as_bytes()).expect("unable to write module docs"); - } - } -} - -impl Renderer for MarkdownRenderer { - fn render(stdlib: &MaslLibrary, output_dir: &str) { - // Write per module markdown file - for module in stdlib.modules() { - let (dir_path, file_path) = get_dir_and_file_paths(module.path().clone(), output_dir); - - // Create the directories if they don't exist - fs::create_dir_all(dir_path).expect("Failed to create directory"); - - let f = fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(file_path) - .expect("unable to open stdlib markdown file"); - - Self::write_docs_module(&f, module); - Self::write_docs_header(&f, module); - for export in module.procedures() { - if !export.visibility().is_exported() { - continue; - } - if let Export::Alias(ref reexported) = export { - Self::write_docs_reexported_proc(&f, reexported); - } - } - for export in module.procedures() { - if !export.visibility().is_exported() { - continue; - } - if let Export::Procedure(ref proc) = export { - Self::write_docs_procedure(&f, proc); - } - } - } - } -} - -// HELPER FUNCTIONS -// ================================================================================================ - -fn get_dir_and_file_paths(mut path: LibraryPath, output_dir: &str) -> (PathBuf, PathBuf) { - let file_name = path.pop().unwrap(); - let dir_path = path - .components() - .skip(1) - .fold(PathBuf::from(output_dir), |acc, component| acc.join(component.as_ref())); - let file_path = dir_path.join(format!("{}.md", file_name)); - (dir_path, file_path) -} diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 8b6a7948b1..e28c44847b 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -2,18 +2,21 @@ extern crate alloc; -use assembly::{ - ast::Module, utils::Deserializable, Library, LibraryNamespace, LibraryPath, MaslLibrary, - Version, -}; +use assembly::{library::CompiledLibrary, utils::Deserializable}; // STANDARD LIBRARY // ================================================================================================ /// TODO: add docs -pub struct StdLibrary(MaslLibrary); +pub struct StdLibrary(CompiledLibrary); -impl From for MaslLibrary { +impl AsRef for StdLibrary { + fn as_ref(&self) -> &CompiledLibrary { + &self.0 + } +} + +impl From for CompiledLibrary { fn from(value: StdLibrary) -> Self { value.0 } @@ -22,42 +25,27 @@ impl From for MaslLibrary { impl Default for StdLibrary { fn default() -> Self { let bytes = include_bytes!(concat!(env!("OUT_DIR"), "/assets/std.masl")); - let contents = MaslLibrary::read_from_bytes(bytes).expect("failed to read std masl!"); + let contents = CompiledLibrary::read_from_bytes(bytes).expect("failed to read std masl!"); Self(contents) } } -impl Library for StdLibrary { - fn root_ns(&self) -> &LibraryNamespace { - self.0.root_ns() - } - - fn version(&self) -> &Version { - self.0.version() - } +#[cfg(test)] +mod tests { + use assembly::LibraryPath; - fn modules(&self) -> impl ExactSizeIterator + '_ { - self.0.modules() - } + use super::*; - fn dependencies(&self) -> &[assembly::LibraryNamespace] { - self.0.dependencies() - } + #[test] + fn test_compile() { + let path = "std::math::u64::overflowing_add".parse::().unwrap(); + let stdlib = StdLibrary::default(); + let exists = stdlib.0.module_infos().any(|module| { + module + .procedure_infos() + .any(|(_, proc)| module.path().clone().append(&proc.name).unwrap() == path) + }); - fn get_module(&self, path: &LibraryPath) -> Option<&Module> { - self.0.get_module(path) + assert!(exists); } } - -#[test] -fn test_compile() { - let path = "std::math::u64::overflowing_add".parse::().unwrap(); - let stdlib = StdLibrary::default(); - let exists = stdlib.modules().any(|module| { - module - .procedures() - .any(|proc| module.path().clone().append(proc.name()).unwrap() == path) - }); - - assert!(exists); -} diff --git a/stdlib/tests/crypto/falcon.rs b/stdlib/tests/crypto/falcon.rs index 0220d143e9..2cbf89b6a8 100644 --- a/stdlib/tests/crypto/falcon.rs +++ b/stdlib/tests/crypto/falcon.rs @@ -200,7 +200,7 @@ fn falcon_prove_verify() { let (source, op_stack, _, _, advice_map) = generate_test(sk, message); let program: Program = Assembler::default() - .with_library(&StdLibrary::default()) + .with_compiled_library(StdLibrary::default()) .expect("failed to load stdlib") .assemble_program(source) .expect("failed to compile test source"); diff --git a/stdlib/tests/mem/mod.rs b/stdlib/tests/mem/mod.rs index 95c75bf11e..b32794a71d 100644 --- a/stdlib/tests/mem/mod.rs +++ b/stdlib/tests/mem/mod.rs @@ -22,17 +22,21 @@ fn test_memcopy() { end "; + let stdlib = StdLibrary::default(); let assembler = assembly::Assembler::default() - .with_library(&StdLibrary::default()) + .with_compiled_library(&stdlib) .expect("failed to load stdlib"); let program: Program = assembler.assemble_program(source).expect("Failed to compile test source."); + let mut host = DefaultHost::default(); + host.load_mast_forest(stdlib.as_ref().mast_forest().clone()); + let mut process = Process::new( program.kernel().clone(), StackInputs::default(), - DefaultHost::default(), + host, ExecutionOptions::default(), ); process.execute(&program).unwrap(); diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index 0a717511c6..4ae868ddb4 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -8,12 +8,16 @@ extern crate std; // IMPORTS // ================================================================================================ -use processor::Program; +use assembly::library::CompiledLibrary; +use processor::{MastForest, Program}; + #[cfg(not(target_family = "wasm"))] use proptest::prelude::{Arbitrary, Strategy}; +#[cfg(not(target_family = "wasm"))] +use alloc::format; + use alloc::{ - format, string::{String, ToString}, sync::Arc, vec::Vec, @@ -25,7 +29,7 @@ use vm_core::{chiplets::hasher::apply_permutation, ProgramInfo}; pub use assembly::{ diagnostics::{Report, SourceFile}, - Library, LibraryPath, MaslLibrary, + LibraryPath, }; pub use pretty_assertions::{assert_eq, assert_ne, assert_str_eq}; pub use processor::{ @@ -173,11 +177,11 @@ macro_rules! assert_assembler_diagnostic { /// ExecutionError which contains the specified substring. pub struct Test { pub source: Arc, - pub kernel: Option, + pub kernel_source: Option, pub stack_inputs: StackInputs, pub advice_inputs: AdviceInputs, pub in_debug_mode: bool, - pub libraries: Vec, + pub libraries: Vec, pub add_modules: Vec<(LibraryPath, String)>, } @@ -189,7 +193,7 @@ impl Test { pub fn new(name: &str, source: &str, in_debug_mode: bool) -> Self { Test { source: Arc::new(SourceFile::new(name, source.to_string())), - kernel: None, + kernel_source: None, stack_inputs: StackInputs::default(), advice_inputs: AdviceInputs::default(), in_debug_mode, @@ -218,6 +222,7 @@ impl Test { /// Executes the test and validates that the process memory has the elements of `expected_mem` /// at address `mem_start_addr` and that the end of the stack execution trace matches the /// `final_stack`. + #[track_caller] pub fn expect_stack_and_memory( &self, final_stack: &[u64], @@ -225,8 +230,14 @@ impl Test { expected_mem: &[u64], ) { // compile the program - let program: Program = self.compile().expect("Failed to compile test source."); - let host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + let (program, kernel) = self.compile().expect("Failed to compile test source."); + let mut host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + if let Some(kernel) = kernel { + host.load_mast_forest(kernel); + } + for library in &self.libraries { + host.load_mast_forest(library.mast_forest().clone()); + } // execute the test let mut process = Process::new( @@ -275,16 +286,18 @@ impl Test { // -------------------------------------------------------------------------------------------- /// Compiles a test's source and returns the resulting Program or Assembly error. - pub fn compile(&self) -> Result { - use assembly::{ast::ModuleKind, CompileOptions}; - #[allow(unused)] - let assembler = if let Some(kernel) = self.kernel.as_ref() { - // TODO: Load in kernel after we add the new `Assembler::add_library()` - assembly::Assembler::default() + pub fn compile(&self) -> Result<(Program, Option), Report> { + use assembly::{ast::ModuleKind, Assembler, CompileOptions}; + + let (assembler, compiled_kernel) = if let Some(kernel) = self.kernel_source.as_ref() { + let kernel_lib = Assembler::default().assemble_kernel(kernel).unwrap(); + let compiled_kernel = kernel_lib.mast_forest().clone(); + + (Assembler::with_kernel(kernel_lib), Some(compiled_kernel)) } else { - assembly::Assembler::default() + (Assembler::default(), None) }; - let assembler = self + let mut assembler = self .add_modules .iter() .fold(assembler, |assembler, (path, source)| { @@ -295,19 +308,26 @@ impl Test { ) .expect("invalid masm source code") }) - .with_debug_mode(self.in_debug_mode) - .with_libraries(self.libraries.iter()) - .expect("failed to load stdlib"); + .with_debug_mode(self.in_debug_mode); + for library in &self.libraries { + assembler.add_compiled_library(library).unwrap(); + } - assembler.assemble_program(self.source.clone()) + Ok((assembler.assemble_program(self.source.clone())?, compiled_kernel)) } /// Compiles the test's source to a Program and executes it with the tests inputs. Returns a /// resulting execution trace or error. #[track_caller] pub fn execute(&self) -> Result { - let program: Program = self.compile().expect("Failed to compile test source."); - let host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + let (program, kernel) = self.compile().expect("Failed to compile test source."); + let mut host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + if let Some(kernel) = kernel { + host.load_mast_forest(kernel); + } + for library in &self.libraries { + host.load_mast_forest(library.mast_forest().clone()); + } processor::execute(&program, self.stack_inputs.clone(), host, ExecutionOptions::default()) } @@ -316,8 +336,15 @@ impl Test { pub fn execute_process( &self, ) -> Result>, ExecutionError> { - let program: Program = self.compile().expect("Failed to compile test source."); - let host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + let (program, kernel) = self.compile().expect("Failed to compile test source."); + let mut host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + if let Some(kernel) = kernel { + host.load_mast_forest(kernel); + } + for library in &self.libraries { + host.load_mast_forest(library.mast_forest().clone()); + } + let mut process = Process::new( program.kernel().clone(), self.stack_inputs.clone(), @@ -333,8 +360,14 @@ impl Test { /// is true, this function will force a failure by modifying the first output. pub fn prove_and_verify(&self, pub_inputs: Vec, test_fail: bool) { let stack_inputs = StackInputs::try_from_ints(pub_inputs).unwrap(); - let program: Program = self.compile().expect("Failed to compile test source."); - let host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + let (program, kernel) = self.compile().expect("Failed to compile test source."); + let mut host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + if let Some(kernel) = kernel { + host.load_mast_forest(kernel); + } + for library in &self.libraries { + host.load_mast_forest(library.mast_forest().clone()); + } let (mut stack_outputs, proof) = prover::prove(&program, stack_inputs.clone(), host, ProvingOptions::default()).unwrap(); @@ -352,8 +385,14 @@ impl Test { /// VmStateIterator that allows us to iterate through each clock cycle and inspect the process /// state. pub fn execute_iter(&self) -> VmStateIterator { - let program: Program = self.compile().expect("Failed to compile test source."); - let host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + let (program, kernel) = self.compile().expect("Failed to compile test source."); + let mut host = DefaultHost::new(MemAdviceProvider::from(self.advice_inputs.clone())); + if let Some(kernel) = kernel { + host.load_mast_forest(kernel); + } + for library in &self.libraries { + host.load_mast_forest(library.mast_forest().clone()); + } processor::execute_iter(&program, self.stack_inputs.clone(), host) } diff --git a/test-utils/src/test_builders.rs b/test-utils/src/test_builders.rs index 4cc6fde43d..79a54088fd 100644 --- a/test-utils/src/test_builders.rs +++ b/test-utils/src/test_builders.rs @@ -92,7 +92,7 @@ macro_rules! build_test_by_mode { name, ::alloc::string::String::from($source), )), - kernel: None, + kernel_source: None, stack_inputs, advice_inputs, in_debug_mode: $in_debug_mode, @@ -118,7 +118,7 @@ macro_rules! build_test_by_mode { name, ::alloc::string::String::from($source), )), - kernel: None, + kernel_source: None, stack_inputs, advice_inputs, in_debug_mode: $in_debug_mode, @@ -143,7 +143,7 @@ macro_rules! build_test_by_mode { name, String::from($source), )), - kernel: None, + kernel_source: None, stack_inputs, advice_inputs, in_debug_mode: $in_debug_mode, @@ -167,7 +167,7 @@ macro_rules! build_test_by_mode { name, ::alloc::string::String::from($source), )), - kernel: None, + kernel_source: None, stack_inputs, advice_inputs, in_debug_mode: $in_debug_mode,