From b8a767f3dc318db7b36502faec77dc6987bdeafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Laferri=C3=A8re?= Date: Thu, 25 Jul 2024 11:50:23 -0400 Subject: [PATCH] add `Assembler::assemble_library()` (#1413) * add `Assembler::assemble_library()` * changelog * fix `no_std` * add test * `MastForestStore::procedure_digests()` * assemble_graph: fix comment * fix weird formatting * remove `Assembler::assemble_graph()` * remove `ModuleGraph.topo` * Rename `Assembler::assemble` to `assemble_program` * Remove `ModuleInfo` and `ProcedureInfo` * fix docs * fix docs --- CHANGELOG.md | 1 + assembly/README.md | 8 +-- assembly/src/assembler/mod.rs | 58 ++++++++++++++++++- assembly/src/assembler/module_graph/mod.rs | 8 +-- assembly/src/assembler/tests.rs | 8 +-- assembly/src/library/error.rs | 10 ++++ assembly/src/library/mod.rs | 54 ++++++++++++++++- assembly/src/testing.rs | 2 +- assembly/src/tests.rs | 51 +++++++++++++++- core/src/mast/mod.rs | 7 ++- miden/README.md | 6 +- miden/benches/program_compilation.rs | 2 +- miden/benches/program_execution.rs | 3 +- miden/src/cli/data.rs | 5 +- miden/src/examples/blake3.rs | 2 +- miden/src/examples/fibonacci.rs | 2 +- miden/src/repl/mod.rs | 2 +- miden/src/tools/mod.rs | 2 +- .../operations/decorators/events.rs | 4 +- processor/src/host/mast_forest_store.rs | 4 +- stdlib/tests/crypto/falcon.rs | 2 +- stdlib/tests/crypto/stark/mod.rs | 2 +- stdlib/tests/mem/mod.rs | 3 +- test-utils/src/lib.rs | 2 +- 24 files changed, 205 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f71356af6d..2ecdb69af1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ - Added functions to `MastForestBuilder` to allow ensuring of nodes with fewer LOC (#1404) - Make `Assembler` single-use (#1409) - Remove `ProcedureCache` from the assembler (#1411). +- Add `Assembler::assemble_library()` (#1413) #### Changed diff --git a/assembly/README.md b/assembly/README.md index 9a76a116a8..2814bb79b5 100644 --- a/assembly/README.md +++ b/assembly/README.md @@ -28,10 +28,10 @@ use miden_assembly::Assembler; let assembler = Assembler::default(); // Emit a program which pushes values 3 and 5 onto the stack and adds them -let program = assembler.assemble("begin push.3 push.5 add end").unwrap(); +let program = assembler.assemble_program("begin push.3 push.5 add end").unwrap(); // Emit a program from some source code on disk (requires the `std` feature) -let program = assembler.assemble(&Path::new("./example.masm")).unwrap(); +let program = assembler.assemble_program(&Path::new("./example.masm")).unwrap(); ``` > [!NOTE] @@ -119,7 +119,7 @@ Programs compiled by this assembler will be able to make calls to the `foo` procedure by executing the `syscall` instruction, like so: ```rust -assembler.assemble(" +assembler.assemble_program(" begin syscall.foo end @@ -169,7 +169,7 @@ let assembler = Assembler::default() .unwrap(); // Assemble our program -assembler.assemble(" +assembler.assemble_program(" begin push.1.2 syscall.foo diff --git a/assembly/src/assembler/mod.rs b/assembly/src/assembler/mod.rs index c49e121e68..2b68caa124 100644 --- a/assembly/src/assembler/mod.rs +++ b/assembly/src/assembler/mod.rs @@ -1,8 +1,10 @@ use crate::{ ast::{ self, FullyQualifiedProcedureName, Instruction, InvocationTarget, InvokeKind, ModuleKind, + ProcedureIndex, }, diagnostics::Report, + library::CompiledLibrary, sema::SemanticAnalysisError, AssemblyError, Compile, CompileOptions, Felt, Library, LibraryNamespace, LibraryPath, RpoDigest, Spanned, ONE, ZERO, @@ -41,11 +43,12 @@ use self::module_graph::{CallerInfo, ModuleGraph, ResolvedTarget}; /// Programs compiled with an empty kernel cannot use the `syscall` instruction. /// /// -/// * If you have a single executable module you want to compile, just call [Assembler::assemble]. +/// * If you have a single executable module you want to compile, just call +/// [Assembler::assemble_program]. /// * If you want to link your executable to a few other modules that implement supporting /// procedures, build the assembler with them first, using the various builder methods on /// [Assembler], e.g. [Assembler::with_module], [Assembler::with_library], etc. Then, call -/// [Assembler::assemble] to get your compiled program. +/// [Assembler::assemble_program] to get your compiled program. #[derive(Clone, Default)] pub struct Assembler { /// The global [ModuleGraph] for this assembler. @@ -225,6 +228,55 @@ impl Assembler { // ------------------------------------------------------------------------------------------------ /// Compilation/Assembly impl Assembler { + /// Assembles a set of modules into a library. + pub fn assemble_library( + mut self, + modules: impl Iterator, + ) -> Result { + let module_indices: Vec = modules + .map(|module| { + let module = module.compile_with_options(CompileOptions::for_library())?; + + Ok(self.module_graph.add_module(module)?) + }) + .collect::>()?; + self.module_graph.recompute()?; + + let mut mast_forest_builder = MastForestBuilder::default(); + + let exports = { + let mut exports = Vec::new(); + + for module_idx in module_indices { + let module = self.module_graph.get_module(module_idx).unwrap(); + + for (proc_idx, procedure) in module.procedures().enumerate() { + // Only add exports; locals will be added if they are in the call graph rooted + // at those procedures + if !procedure.visibility().is_exported() { + continue; + } + + let gid = GlobalProcedureIndex { + module: module_idx, + index: ProcedureIndex::new(proc_idx), + }; + + self.compile_subgraph(gid, false, &mut mast_forest_builder)?; + + exports.push(FullyQualifiedProcedureName::new( + module.path().clone(), + procedure.name().clone(), + )); + } + } + + exports + }; + + Ok(CompiledLibrary::new(mast_forest_builder.build(), exports)?) + } + /// Compiles the provided module into a [`Program`]. The resulting program can be executed on /// Miden VM. /// @@ -232,7 +284,7 @@ impl Assembler { /// /// Returns an error if parsing or compilation of the specified program fails, or if the source /// doesn't have an entrypoint. - pub fn assemble(self, source: impl Compile) -> Result { + pub fn assemble_program(self, source: impl Compile) -> Result { let opts = CompileOptions { warnings_as_errors: self.warnings_as_errors, ..CompileOptions::default() diff --git a/assembly/src/assembler/module_graph/mod.rs b/assembly/src/assembler/module_graph/mod.rs index 0ce18226dc..56d96c673d 100644 --- a/assembly/src/assembler/module_graph/mod.rs +++ b/assembly/src/assembler/module_graph/mod.rs @@ -42,8 +42,6 @@ pub struct ModuleGraph { /// The global call graph of calls, not counting those that are performed directly via MAST /// root. callgraph: CallGraph, - /// The computed topological ordering of the call graph - topo: Vec, /// The set of MAST roots which have procedure definitions in this graph. There can be /// multiple procedures bound to the same root due to having identical code. roots: BTreeMap>, @@ -191,9 +189,6 @@ impl ModuleGraph { return Ok(()); } - // Remove previous topological sort, since it is no longer valid - self.topo.clear(); - // Visit all of the pending modules, assigning them ids, and adding them to the module // graph after rewriting any calls to use absolute paths let high_water_mark = self.modules.len(); @@ -283,7 +278,7 @@ impl ModuleGraph { } // Make sure the graph is free of cycles - let topo = self.callgraph.toposort().map_err(|cycle| { + self.callgraph.toposort().map_err(|cycle| { let iter = cycle.into_node_ids(); let mut nodes = Vec::with_capacity(iter.len()); for node in iter { @@ -293,7 +288,6 @@ impl ModuleGraph { } AssemblyError::Cycle { nodes } })?; - self.topo = topo; Ok(()) } diff --git a/assembly/src/assembler/tests.rs b/assembly/src/assembler/tests.rs index 523d9072c5..b0c62bb446 100644 --- a/assembly/src/assembler/tests.rs +++ b/assembly/src/assembler/tests.rs @@ -123,7 +123,7 @@ fn nested_blocks() { syscall.foo end"#; - let program = assembler.assemble(program).unwrap(); + let program = assembler.assemble_program(program).unwrap(); let exec_bar_node_id = { // bar procedure @@ -224,7 +224,7 @@ fn duplicate_procedure() { end "#; - let program = assembler.assemble(program_source).unwrap(); + let program = assembler.assemble_program(program_source).unwrap(); assert_eq!(program.num_procedures(), 2); } @@ -243,7 +243,7 @@ fn duplicate_nodes() { end "#; - let program = assembler.assemble(program_source).unwrap(); + let program = assembler.assemble_program(program_source).unwrap(); let mut expected_mast_forest = MastForest::new(); @@ -339,5 +339,5 @@ fn explicit_fully_qualified_procedure_references() { exec.::foo::baz::baz end"#; - assert_matches!(assembler.assemble(program), Ok(_)); + assert_matches!(assembler.assemble_program(program), Ok(_)); } diff --git a/assembly/src/library/error.rs b/assembly/src/library/error.rs index 5f46d28c0d..fc6a803795 100644 --- a/assembly/src/library/error.rs +++ b/assembly/src/library/error.rs @@ -59,3 +59,13 @@ pub enum LibraryError { #[cfg(feature = "std")] Io(#[from] std::io::Error), } + +#[derive(Debug, thiserror::Error, Diagnostic)] +pub enum CompiledLibraryError { + #[error("Invalid exports: MAST forest has {roots_len} procedure roots, but exports have {exports_len}")] + #[diagnostic()] + InvalidExports { + exports_len: usize, + roots_len: usize, + }, +} diff --git a/assembly/src/library/mod.rs b/assembly/src/library/mod.rs index e474b346b3..184806acce 100644 --- a/assembly/src/library/mod.rs +++ b/assembly/src/library/mod.rs @@ -1,4 +1,8 @@ -use crate::ast; +use alloc::vec::Vec; + +use vm_core::mast::MastForest; + +use crate::ast::{self, FullyQualifiedProcedureName}; mod error; mod masl; @@ -6,7 +10,7 @@ mod namespace; mod path; mod version; -pub use self::error::LibraryError; +pub use self::error::{CompiledLibraryError, LibraryError}; pub use self::masl::MaslLibrary; pub use self::namespace::{LibraryNamespace, LibraryNamespaceError}; pub use self::path::{LibraryPath, LibraryPathComponent, PathError}; @@ -15,6 +19,52 @@ pub use self::version::{Version, VersionError}; #[cfg(test)] mod tests; +// COMPILED LIBRARY +// =============================================================================================== + +/// Represents a library where all modules modules were compiled into a [`MastForest`]. +pub struct CompiledLibrary { + mast_forest: MastForest, + // a path for every `root` in the associated MAST forest + exports: Vec, +} + +/// Constructors +impl CompiledLibrary { + /// Constructs a new [`CompiledLibrary`]. + pub fn new( + mast_forest: MastForest, + exports: Vec, + ) -> Result { + if mast_forest.num_procedures() as usize != exports.len() { + return Err(CompiledLibraryError::InvalidExports { + exports_len: exports.len(), + roots_len: mast_forest.num_procedures() as usize, + }); + } + + Ok(Self { + mast_forest, + exports, + }) + } +} + +impl CompiledLibrary { + /// Returns the inner [`MastForest`]. + pub fn mast_forest(&self) -> &MastForest { + &self.mast_forest + } + + /// Returns the fully qualified name of all procedures exported by the library. + pub fn exports(&self) -> &[FullyQualifiedProcedureName] { + &self.exports + } +} + +// LIBRARY +// =============================================================================================== + /// Maximum number of modules in a library. const MAX_MODULES: usize = u16::MAX as usize; diff --git a/assembly/src/testing.rs b/assembly/src/testing.rs index 9518f3c617..b4f512fa85 100644 --- a/assembly/src/testing.rs +++ b/assembly/src/testing.rs @@ -308,7 +308,7 @@ impl TestContext { /// module represented in `source`. #[track_caller] pub fn assemble(&mut self, source: impl Compile) -> Result { - self.assembler.clone().assemble(source) + self.assembler.clone().assemble_program(source) } /// Compile a module from `source`, with the fully-qualified name `path`, to MAST, returning diff --git a/assembly/src/tests.rs b/assembly/src/tests.rs index 9db5847ddc..765f75adaa 100644 --- a/assembly/src/tests.rs +++ b/assembly/src/tests.rs @@ -6,7 +6,7 @@ use crate::{ diagnostics::Report, regex, source_file, testing::{Pattern, TestContext}, - Assembler, Library, LibraryNamespace, LibraryPath, MaslLibrary, Version, + Assembler, Library, LibraryNamespace, LibraryPath, MaslLibrary, ModuleParser, Version, }; type TestResult = Result<(), Report>; @@ -1495,7 +1495,7 @@ fn program_with_phantom_mast_call() -> TestResult { let ast = context.parse_program(source)?; let assembler = Assembler::default().with_debug_mode(true); - assembler.assemble(ast)?; + assembler.assemble_program(ast)?; Ok(()) } @@ -2363,6 +2363,53 @@ fn invalid_while() -> TestResult { Ok(()) } +#[test] +fn test_compiled_library() { + let mut mod_parser = ModuleParser::new(ModuleKind::Library); + let mod1 = { + let source = source_file!( + " + proc.internal + push.5 + end + export.foo + push.1 + drop + end + export.bar + exec.internal + drop + end + " + ); + mod_parser.parse(LibraryPath::new("mylib::mod1").unwrap(), source).unwrap() + }; + + let mod2 = { + let source = source_file!( + " + export.foo + push.7 + add.5 + end + # Same definition as mod1::foo + export.bar + push.1 + drop + end + " + ); + mod_parser.parse(LibraryPath::new("mylib::mod2").unwrap(), source).unwrap() + }; + + let compiled_library = { + let assembler = Assembler::new(); + assembler.assemble_library(vec![mod1, mod2].into_iter()).unwrap() + }; + + assert_eq!(compiled_library.exports().len(), 4); +} + // DUMMY LIBRARY // ================================================================================================ diff --git a/core/src/mast/mod.rs b/core/src/mast/mod.rs index 20d1ed9309..a6d73fe29a 100644 --- a/core/src/mast/mod.rs +++ b/core/src/mast/mod.rs @@ -95,10 +95,15 @@ impl MastForest { } /// Returns an iterator over the digest of the procedures in this MAST forest. - pub fn procedure_roots(&self) -> impl Iterator + '_ { + pub fn procedure_digests(&self) -> impl Iterator + '_ { self.roots.iter().map(|&root_id| self[root_id].digest()) } + /// Returns an iterator over the IDs of the procedures in this MAST forest. + pub fn procedure_roots(&self) -> &[MastNodeId] { + &self.roots + } + /// Returns the number of procedures in this MAST forest. pub fn num_procedures(&self) -> u32 { self.roots diff --git a/miden/README.md b/miden/README.md index e6f1431a6b..fec94f49a9 100644 --- a/miden/README.md +++ b/miden/README.md @@ -57,7 +57,7 @@ use processor::ExecutionOptions; let mut assembler = Assembler::default(); // compile Miden assembly source code into a program -let program = assembler.assemble("begin push.3 push.5 add end").unwrap(); +let program = assembler.assemble_program("begin push.3 push.5 add end").unwrap(); // use an empty list as initial stack let stack_inputs = StackInputs::default(); @@ -105,7 +105,7 @@ use miden_vm::{Assembler, DefaultHost, ProvingOptions, Program, prove, StackInpu let mut assembler = Assembler::default(); // this is our program, we compile it from assembly code -let program = assembler.assemble("begin push.3 push.5 add end").unwrap(); +let program = assembler.assemble_program("begin push.3 push.5 add end").unwrap(); // let's execute it and generate a STARK proof let (outputs, proof) = prove( @@ -193,7 +193,7 @@ let source = format!( n - 1 ); let mut assembler = Assembler::default(); -let program = assembler.assemble(&source).unwrap(); +let program = assembler.assemble_program(&source).unwrap(); // initialize a default host (with an empty advice provider) let host = DefaultHost::default(); diff --git a/miden/benches/program_compilation.rs b/miden/benches/program_compilation.rs index 95fa9b38ff..4462b7231d 100644 --- a/miden/benches/program_compilation.rs +++ b/miden/benches/program_compilation.rs @@ -18,7 +18,7 @@ fn program_compilation(c: &mut Criterion) { let assembler = Assembler::default() .with_library(&StdLibrary::default()) .expect("failed to load stdlib"); - assembler.assemble(source).expect("Failed to compile test source.") + assembler.assemble_program(source).expect("Failed to compile test source.") }); }); diff --git a/miden/benches/program_execution.rs b/miden/benches/program_execution.rs index 57719f31b2..4487672ca9 100644 --- a/miden/benches/program_execution.rs +++ b/miden/benches/program_execution.rs @@ -18,7 +18,8 @@ fn program_execution(c: &mut Criterion) { let assembler = Assembler::default() .with_library(&StdLibrary::default()) .expect("failed to load stdlib"); - let program: Program = assembler.assemble(source).expect("Failed to compile test source."); + let program: Program = + assembler.assemble_program(source).expect("Failed to compile test source."); bench.iter(|| { execute( &program, diff --git a/miden/src/cli/data.rs b/miden/src/cli/data.rs index 1a62ca7062..76355a3601 100644 --- a/miden/src/cli/data.rs +++ b/miden/src/cli/data.rs @@ -419,8 +419,9 @@ impl ProgramFile { .with_libraries(libraries.into_iter()) .wrap_err("Failed to load libraries")?; - let program: Program = - assembler.assemble(self.ast.as_ref()).wrap_err("Failed to compile program")?; + let program: Program = assembler + .assemble_program(self.ast.as_ref()) + .wrap_err("Failed to compile program")?; Ok(program) } diff --git a/miden/src/examples/blake3.rs b/miden/src/examples/blake3.rs index 2e87cadbcc..7bae853923 100644 --- a/miden/src/examples/blake3.rs +++ b/miden/src/examples/blake3.rs @@ -47,7 +47,7 @@ fn generate_blake3_program(n: usize) -> Program { Assembler::default() .with_library(&StdLibrary::default()) .unwrap() - .assemble(program) + .assemble_program(program) .unwrap() } diff --git a/miden/src/examples/fibonacci.rs b/miden/src/examples/fibonacci.rs index 7bd6555c52..98ea60122c 100644 --- a/miden/src/examples/fibonacci.rs +++ b/miden/src/examples/fibonacci.rs @@ -39,7 +39,7 @@ fn generate_fibonacci_program(n: usize) -> Program { n - 1 ); - Assembler::default().assemble(program).unwrap() + Assembler::default().assemble_program(program).unwrap() } /// Computes the `n`-th term of Fibonacci sequence diff --git a/miden/src/repl/mod.rs b/miden/src/repl/mod.rs index 5d711e81aa..1139f20e42 100644 --- a/miden/src/repl/mod.rs +++ b/miden/src/repl/mod.rs @@ -312,7 +312,7 @@ fn execute( .with_libraries(provided_libraries.iter()) .map_err(|err| format!("{err}"))?; - let program = assembler.assemble(program).map_err(|err| format!("{err}"))?; + let program = assembler.assemble_program(program).map_err(|err| format!("{err}"))?; let stack_inputs = StackInputs::default(); let host = DefaultHost::default(); diff --git a/miden/src/tools/mod.rs b/miden/src/tools/mod.rs index 992e91ddf7..f4718ce014 100644 --- a/miden/src/tools/mod.rs +++ b/miden/src/tools/mod.rs @@ -217,7 +217,7 @@ where let program = Assembler::default() .with_debug_mode(true) .with_library(&StdLibrary::default())? - .assemble(program)?; + .assemble_program(program)?; let mut execution_details = ExecutionDetails::default(); let vm_state_iterator = processor::execute_iter(&program, stack_inputs, host); diff --git a/miden/tests/integration/operations/decorators/events.rs b/miden/tests/integration/operations/decorators/events.rs index c9385fe46f..c70d1c660d 100644 --- a/miden/tests/integration/operations/decorators/events.rs +++ b/miden/tests/integration/operations/decorators/events.rs @@ -13,7 +13,7 @@ fn test_event_handling() { end"; // compile and execute program - let program: Program = Assembler::default().assemble(source).unwrap(); + let program: Program = Assembler::default().assemble_program(source).unwrap(); let mut host = TestHost::default(); processor::execute(&program, Default::default(), &mut host, Default::default()).unwrap(); @@ -33,7 +33,7 @@ fn test_trace_handling() { end"; // compile program - let program: Program = Assembler::default().assemble(source).unwrap(); + let program: Program = Assembler::default().assemble_program(source).unwrap(); let mut host = TestHost::default(); // execute program with disabled tracing diff --git a/processor/src/host/mast_forest_store.rs b/processor/src/host/mast_forest_store.rs index 06c3634250..08718a6c63 100644 --- a/processor/src/host/mast_forest_store.rs +++ b/processor/src/host/mast_forest_store.rs @@ -25,8 +25,8 @@ impl MemMastForestStore { pub fn insert(&mut self, mast_forest: MastForest) { let mast_forest = Arc::new(mast_forest); - for root in mast_forest.procedure_roots() { - self.mast_forests.insert(root, mast_forest.clone()); + for proc_digest in mast_forest.procedure_digests() { + self.mast_forests.insert(proc_digest, mast_forest.clone()); } } } diff --git a/stdlib/tests/crypto/falcon.rs b/stdlib/tests/crypto/falcon.rs index 8a89c6ae83..0220d143e9 100644 --- a/stdlib/tests/crypto/falcon.rs +++ b/stdlib/tests/crypto/falcon.rs @@ -202,7 +202,7 @@ fn falcon_prove_verify() { let program: Program = Assembler::default() .with_library(&StdLibrary::default()) .expect("failed to load stdlib") - .assemble(source) + .assemble_program(source) .expect("failed to compile test source"); let stack_inputs = StackInputs::try_from_ints(op_stack).expect("failed to create stack inputs"); diff --git a/stdlib/tests/crypto/stark/mod.rs b/stdlib/tests/crypto/stark/mod.rs index ac29770e3b..95cb89af80 100644 --- a/stdlib/tests/crypto/stark/mod.rs +++ b/stdlib/tests/crypto/stark/mod.rs @@ -51,7 +51,7 @@ pub fn generate_recursive_verifier_data( source: &str, stack_inputs: Vec, ) -> Result { - let program: Program = Assembler::default().assemble(source).unwrap(); + let program: Program = Assembler::default().assemble_program(source).unwrap(); let stack_inputs = StackInputs::try_from_ints(stack_inputs).unwrap(); let advice_inputs = AdviceInputs::default(); let advice_provider = MemAdviceProvider::from(advice_inputs); diff --git a/stdlib/tests/mem/mod.rs b/stdlib/tests/mem/mod.rs index 1dbef6927b..95c75bf11e 100644 --- a/stdlib/tests/mem/mod.rs +++ b/stdlib/tests/mem/mod.rs @@ -26,7 +26,8 @@ fn test_memcopy() { .with_library(&StdLibrary::default()) .expect("failed to load stdlib"); - let program: Program = assembler.assemble(source).expect("Failed to compile test source."); + let program: Program = + assembler.assemble_program(source).expect("Failed to compile test source."); let mut process = Process::new( program.kernel().clone(), diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index 48d29d0f39..0a717511c6 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -299,7 +299,7 @@ impl Test { .with_libraries(self.libraries.iter()) .expect("failed to load stdlib"); - assembler.assemble(self.source.clone()) + assembler.assemble_program(self.source.clone()) } /// Compiles the test's source to a Program and executes it with the tests inputs. Returns a