diff --git a/CHANGELOG.md b/CHANGELOG.md index 967917ab3c..1d1545d354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,11 +27,19 @@ - Added `Assembler::assemble_library()` (#1413). - Added `Assembler::assemble_kernel()` (#1418). - Added `miden_core::prettier::pretty_print_csv` helper, for formatting of iterators over `PrettyPrint` values as comma-separated items +- Added source location tracking to assembled MAST (#1419) +- Added source code management primitives in `miden-core` (#1419) +- Added `make test-fast` and `make test-skip-proptests` Makefile targets for faster testing during local development +- Added `ProgramFile::read_with` constructor that takes a `SourceManager` impl to use for source management #### Changed - When using `if.(true|false) .. end`, the parser used to emit an empty block for the branch that was elided. The parser now emits a block containing a single `nop` instruction instead, which is equivalent to the code emitted by the assembler when lowering to MAST. - `internals` configuration feature was renamed to `testing` (#1399). +- The `AssemblyOp` decorator now contains an optional `Location` (#1419) +- The `Assembler` now requires passing in a `Arc`, for use in rendering diagnostics +- The `Module::parse_file` and `Module::parse_str` functions have been removed in favor of calling `Module::parser` and then using the `ModuleParser` methods +- The `Compile` trait now requires passing a `SourceManager` reference along with the item to be compiled ## 0.9.2 (2024-05-22) - `stdlib` crate only diff --git a/Cargo.lock b/Cargo.lock index 6dc689e1b8..80b44aed47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -679,6 +679,19 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "generator" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "979f00864edc7516466d6b3157706e06c032f22715700ddd878228a91d02bc56" +dependencies = [ + "cfg-if", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -926,6 +939,19 @@ dependencies = [ "log", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -999,10 +1025,15 @@ dependencies = [ name = "miden-core" version = "0.10.0" dependencies = [ + "lock_api", + "loom", + "memchr", "miden-crypto", "miden-formatting", + "miette", "num-derive", "num-traits", + "parking_lot", "proptest", "thiserror", "winter-math 0.9.0", @@ -1764,6 +1795,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -2413,6 +2450,70 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Makefile b/Makefile index 8ad16f81e1..e46a8f8c74 100644 --- a/Makefile +++ b/Makefile @@ -50,9 +50,21 @@ mdbook: ## Generates mdbook documentation # --- testing ------------------------------------------------------------------------------------- .PHONY: test -test: ## Runs all tests +test: ## Runs all tests with the release profile $(DEBUG_ASSERTIONS) cargo nextest run --cargo-profile test-release --features testing +.PHONY: test-fast +test-fast: ## Runs all tests with the debug profile + $(DEBUG_ASSERTIONS) cargo nextest run --features testing + +.PHONY: test-skip-proptests +test-skip-proptests: ## Runs all tests, except property-based tests + $(DEBUG_ASSERTIONS) cargo nextest run --features testing -E 'not test(#*proptest)' + +.PHONY: test-loom +test-loom: ## Runs all loom-based tests + RUSTFLAGS="--cfg loom" cargo nextest run --cargo-profile test-release --features testing -E 'test(#*loom)' + # --- checking ------------------------------------------------------------------------------------ .PHONY: check diff --git a/assembly/Cargo.toml b/assembly/Cargo.toml index 3ba27714bd..bc74637355 100644 --- a/assembly/Cargo.toml +++ b/assembly/Cargo.toml @@ -49,7 +49,9 @@ tracing = { version = "0.1", default-features = false, features = [ ] } thiserror = { version = "1.0", git = "https://github.com/bitwalker/thiserror", branch = "no-std", default-features = false } unicode-width = { version = "0.1", features = ["no_std"] } -vm-core = { package = "miden-core", path = "../core", version = "0.10", default-features = false } +vm-core = { package = "miden-core", path = "../core", version = "0.10", default-features = false, features = [ + "diagnostics", +] } [dev-dependencies] pretty_assertions = "1.4" diff --git a/assembly/src/assembler/basic_block_builder.rs b/assembly/src/assembler/basic_block_builder.rs index dfd4ee7af6..fff02d4357 100644 --- a/assembly/src/assembler/basic_block_builder.rs +++ b/assembly/src/assembler/basic_block_builder.rs @@ -1,4 +1,4 @@ -use crate::{ast::Instruction, AssemblyError}; +use crate::{ast::Instruction, AssemblyError, Span}; use super::{ mast_forest_builder::MastForestBuilder, BodyWrapper, Decorator, DecoratorList, ProcedureContext, @@ -83,12 +83,18 @@ impl BasicBlockBuilder { /// /// This indicates that the provided instruction should be tracked and the cycle count for /// this instruction will be computed when the call to set_instruction_cycle_count() is made. - pub fn track_instruction(&mut self, instruction: &Instruction, proc_ctx: &ProcedureContext) { + pub fn track_instruction( + &mut self, + instruction: &Span, + proc_ctx: &ProcedureContext, + ) { + let span = instruction.span(); + let location = proc_ctx.source_manager().location(span).ok(); let context_name = proc_ctx.name().to_string(); let num_cycles = 0; let op = instruction.to_string(); let should_break = instruction.should_break(); - let op = AssemblyOp::new(context_name, num_cycles, op, should_break); + let op = AssemblyOp::new(location, context_name, num_cycles, op, should_break); self.push_decorator(Decorator::AsmOp(op)); self.last_asmop_pos = self.decorators.len() - 1; } diff --git a/assembly/src/assembler/instruction/env_ops.rs b/assembly/src/assembler/instruction/env_ops.rs index f6260ca671..358df6b55e 100644 --- a/assembly/src/assembler/instruction/env_ops.rs +++ b/assembly/src/assembler/instruction/env_ops.rs @@ -1,5 +1,5 @@ use super::{mem_ops::local_to_absolute_addr, push_felt, BasicBlockBuilder}; -use crate::{assembler::ProcedureContext, AssemblyError, Felt, Spanned}; +use crate::{assembler::ProcedureContext, AssemblyError, Felt, SourceSpan}; use vm_core::Operation::*; // CONSTANT INPUTS @@ -54,11 +54,12 @@ pub fn locaddr( pub fn caller( span: &mut BasicBlockBuilder, proc_ctx: &ProcedureContext, + source_span: SourceSpan, ) -> Result<(), AssemblyError> { if !proc_ctx.is_kernel() { return Err(AssemblyError::CallerOutsideOfKernel { - span: proc_ctx.span(), - source_file: proc_ctx.source_file(), + span: source_span, + source_file: proc_ctx.source_manager().get(source_span.source_id()).ok(), }); } span.push_op(Caller); diff --git a/assembly/src/assembler/instruction/field_ops.rs b/assembly/src/assembler/instruction/field_ops.rs index 2e6cd2af0e..c5a47714a3 100644 --- a/assembly/src/assembler/instruction/field_ops.rs +++ b/assembly/src/assembler/instruction/field_ops.rs @@ -93,8 +93,9 @@ pub fn div_imm( imm: Span, ) -> Result<(), AssemblyError> { if imm == ZERO { - let source_file = proc_ctx.source_file(); - let error = Report::new(crate::parser::ParsingError::DivisionByZero { span: imm.span() }); + let source_span = imm.span(); + let source_file = proc_ctx.source_manager().get(source_span.source_id()).ok(); + let error = Report::new(crate::parser::ParsingError::DivisionByZero { span: source_span }); return Err(if let Some(source_file) = source_file { AssemblyError::Other(RelatedError::new(error.with_source_code(source_file))) } else { diff --git a/assembly/src/assembler/instruction/mod.rs b/assembly/src/assembler/instruction/mod.rs index 854b283dd6..abc78241ed 100644 --- a/assembly/src/assembler/instruction/mod.rs +++ b/assembly/src/assembler/instruction/mod.rs @@ -2,7 +2,9 @@ use super::{ ast::InvokeKind, mast_forest_builder::MastForestBuilder, Assembler, BasicBlockBuilder, Felt, Operation, ProcedureContext, }; -use crate::{ast::Instruction, diagnostics::Report, utils::bound_into_included_u64, AssemblyError}; +use crate::{ + ast::Instruction, diagnostics::Report, utils::bound_into_included_u64, AssemblyError, Span, +}; use core::ops::RangeBounds; use vm_core::{mast::MastNodeId, Decorator, ONE, ZERO}; @@ -21,7 +23,7 @@ use self::u32_ops::U32OpMode::*; impl Assembler { pub(super) fn compile_instruction( &self, - instruction: &Instruction, + instruction: &Span, span_builder: &mut BasicBlockBuilder, proc_ctx: &mut ProcedureContext, mast_forest_builder: &mut MastForestBuilder, @@ -50,14 +52,14 @@ impl Assembler { fn compile_instruction_impl( &self, - instruction: &Instruction, + instruction: &Span, span_builder: &mut BasicBlockBuilder, proc_ctx: &mut ProcedureContext, mast_forest_builder: &mut MastForestBuilder, ) -> Result, AssemblyError> { use Operation::*; - match instruction { + match &**instruction { Instruction::Nop => span_builder.push_op(Noop), Instruction::Assert => span_builder.push_op(Assert(0)), Instruction::AssertWithError(err_code) => { @@ -311,7 +313,7 @@ impl Assembler { Instruction::PushU32List(imms) => env_ops::push_many(imms, span_builder), Instruction::PushFeltList(imms) => env_ops::push_many(imms, span_builder), Instruction::Sdepth => span_builder.push_op(SDepth), - Instruction::Caller => env_ops::caller(span_builder, proc_ctx)?, + Instruction::Caller => env_ops::caller(span_builder, proc_ctx, instruction.span())?, Instruction::Clk => span_builder.push_op(Clk), Instruction::AdvPipe => span_builder.push_op(Pipe), Instruction::AdvPush(n) => adv_ops::adv_push(span_builder, n.expect_value())?, diff --git a/assembly/src/assembler/instruction/procedures.rs b/assembly/src/assembler/instruction/procedures.rs index 3aad75c5f3..951eaf26ad 100644 --- a/assembly/src/assembler/instruction/procedures.rs +++ b/assembly/src/assembler/instruction/procedures.rs @@ -31,7 +31,7 @@ impl Assembler { mast_forest_builder: &mut MastForestBuilder, ) -> Result, AssemblyError> { // Get the procedure from the assembler - let current_source_file = proc_ctx.source_file(); + let current_source_file = self.source_manager.get(span.source_id()).ok(); // If the procedure is cached, register the call to ensure the callset // is updated correctly. diff --git a/assembly/src/assembler/instruction/u32_ops.rs b/assembly/src/assembler/instruction/u32_ops.rs index 4633d91799..037187a9f0 100644 --- a/assembly/src/assembler/instruction/u32_ops.rs +++ b/assembly/src/assembler/instruction/u32_ops.rs @@ -372,9 +372,9 @@ fn handle_division( ) -> Result<(), AssemblyError> { if let Some(imm) = imm { if imm == 0 { - let source_file = proc_ctx.source_file(); - let error = - Report::new(crate::parser::ParsingError::DivisionByZero { span: imm.span() }); + let imm_span = imm.span(); + let source_file = proc_ctx.source_manager().get(imm_span.source_id()).ok(); + let error = Report::new(crate::parser::ParsingError::DivisionByZero { span: imm_span }); return if let Some(source_file) = source_file { Err(AssemblyError::Other(RelatedError::new(error.with_source_code(source_file)))) } else { diff --git a/assembly/src/assembler/mod.rs b/assembly/src/assembler/mod.rs index a262434f86..5469fc82b8 100644 --- a/assembly/src/assembler/mod.rs +++ b/assembly/src/assembler/mod.rs @@ -3,9 +3,10 @@ use crate::{ diagnostics::Report, library::{CompiledLibrary, KernelLibrary}, sema::SemanticAnalysisError, - AssemblyError, Compile, CompileOptions, LibraryNamespace, LibraryPath, RpoDigest, Spanned, + AssemblyError, Compile, CompileOptions, LibraryNamespace, LibraryPath, RpoDigest, + SourceManager, Spanned, }; -use alloc::{collections::BTreeMap, vec::Vec}; +use alloc::{collections::BTreeMap, sync::Arc, vec::Vec}; use mast_forest_builder::MastForestBuilder; use module_graph::{ProcedureWrapper, WrappedModule}; use vm_core::{mast::MastNodeId, Decorator, DecoratorList, Felt, Kernel, Operation, Program}; @@ -46,8 +47,10 @@ use self::module_graph::{CallerInfo, ModuleGraph, ResolvedTarget}; /// procedures, build the assembler with them first, using the various builder methods on /// [Assembler], e.g. [Assembler::with_module], [Assembler::with_compiled_library], etc. Then, /// call [Assembler::assemble_program] to get your compiled program. -#[derive(Clone, Default)] +#[derive(Clone)] pub struct Assembler { + /// The source manager to use for compilation and source location information + source_manager: Arc, /// The global [ModuleGraph] for this assembler. module_graph: ModuleGraph, /// Whether to treat warning diagnostics as errors @@ -56,14 +59,40 @@ pub struct Assembler { in_debug_mode: bool, } +impl Default for Assembler { + fn default() -> Self { + let source_manager = Arc::new(crate::DefaultSourceManager::default()); + let module_graph = ModuleGraph::new(source_manager.clone()); + Self { + source_manager, + module_graph, + warnings_as_errors: false, + in_debug_mode: false, + } + } +} + // ------------------------------------------------------------------------------------------------ /// Constructors impl Assembler { + /// Start building an [Assembler] + pub fn new(source_manager: Arc) -> Self { + let module_graph = ModuleGraph::new(source_manager.clone()); + Self { + source_manager, + module_graph, + warnings_as_errors: false, + in_debug_mode: false, + } + } + /// Start building an [`Assembler`] with a kernel defined by the provided [KernelLibrary]. - pub fn with_kernel(kernel_lib: KernelLibrary) -> Self { + pub fn with_kernel(source_manager: Arc, kernel_lib: KernelLibrary) -> Self { let (kernel, kernel_module, _) = kernel_lib.into_parts(); + let module_graph = ModuleGraph::with_kernel(source_manager.clone(), kernel, kernel_module); Self { - module_graph: ModuleGraph::with_kernel(kernel, kernel_module), + source_manager, + module_graph, ..Default::default() } } @@ -82,6 +111,11 @@ impl Assembler { self } + /// Sets the debug mode flag of the assembler + pub fn set_debug_mode(&mut self, yes: bool) { + self.in_debug_mode = yes; + } + /// Adds `module` to the module graph of the assembler. /// /// The given module must be a library module, or an error will be returned. @@ -129,7 +163,7 @@ impl Assembler { )); } - let module = module.compile_with_options(options)?; + let module = module.compile_with_options(&self.source_manager, options)?; assert_eq!(module.kind(), kind, "expected module kind to match compilation options"); self.module_graph.add_ast_module(module)?; @@ -200,7 +234,7 @@ impl Assembler { let ast_module_indices = modules.into_iter().try_fold(Vec::default(), |mut acc, module| { module - .compile_with_options(CompileOptions::for_library()) + .compile_with_options(&self.source_manager, CompileOptions::for_library()) .and_then(|module| { self.module_graph.add_ast_module(module).map_err(Report::from) }) @@ -251,7 +285,7 @@ impl Assembler { path: Some(LibraryPath::from(LibraryNamespace::Kernel)), }; - let module = module.compile_with_options(options)?; + let module = module.compile_with_options(&self.source_manager, options)?; let module_idx = self.module_graph.add_ast_module(module)?; self.module_graph.recompute()?; @@ -293,7 +327,7 @@ impl Assembler { path: Some(LibraryPath::from(LibraryNamespace::Exec)), }; - let program = source.compile_with_options(options)?; + let program = source.compile_with_options(&self.source_manager, options)?; assert!(program.is_executable()); // Recompute graph with executable module, and start compiling @@ -377,6 +411,7 @@ impl Assembler { // compile. WrappedModule::Info(_) => continue, }; + let export = &module[procedure_gid.index]; match export { Export::Procedure(proc) => { @@ -386,10 +421,14 @@ impl Assembler { module: module.path().clone(), name: proc.name().clone(), }; - let pctx = ProcedureContext::new(procedure_gid, name, proc.visibility()) - .with_num_locals(num_locals) - .with_span(proc.span()) - .with_source_file(proc.source_file()); + let pctx = ProcedureContext::new( + procedure_gid, + name, + proc.visibility(), + self.source_manager.clone(), + ) + .with_num_locals(num_locals) + .with_span(proc.span()); // Compile this procedure let procedure = self.compile_procedure(pctx, mast_forest_builder)?; @@ -404,9 +443,13 @@ impl Assembler { module: module.path().clone(), name: proc_alias.name().clone(), }; - let pctx = ProcedureContext::new(procedure_gid, name, ast::Visibility::Public) - .with_span(proc_alias.span()) - .with_source_file(proc_alias.source_file()); + let pctx = ProcedureContext::new( + procedure_gid, + name, + ast::Visibility::Public, + self.source_manager.clone(), + ) + .with_span(proc_alias.span()); let proc_alias_root = self.resolve_target( InvokeKind::ProcRef, @@ -562,7 +605,6 @@ impl Assembler { ) -> Result { let caller = CallerInfo { span: target.span(), - source_file: proc_ctx.source_file(), module: proc_ctx.id().module, kind, }; diff --git a/assembly/src/assembler/module_graph/analysis/rewrite_check.rs b/assembly/src/assembler/module_graph/analysis/rewrite_check.rs index c2941a181e..0d2424a345 100644 --- a/assembly/src/assembler/module_graph/analysis/rewrite_check.rs +++ b/assembly/src/assembler/module_graph/analysis/rewrite_check.rs @@ -1,4 +1,3 @@ -use alloc::sync::Arc; use core::ops::ControlFlow; use crate::{ @@ -7,7 +6,6 @@ use crate::{ ModuleIndex, ResolvedTarget, }, ast::{visit::Visit, InvocationTarget, InvokeKind, Module}, - diagnostics::SourceFile, AssemblyError, Spanned, }; @@ -38,7 +36,6 @@ impl<'a, 'b: 'a> MaybeRewriteCheck<'a, 'b> { let mut visitor = RewriteCheckVisitor { resolver: self.resolver, module_id, - source_file: module.source_file(), }; match visitor.visit_module(module) { ControlFlow::Break(result) => result, @@ -53,7 +50,6 @@ impl<'a, 'b: 'a> MaybeRewriteCheck<'a, 'b> { struct RewriteCheckVisitor<'a, 'b: 'a> { resolver: &'a NameResolver<'b>, module_id: ModuleIndex, - source_file: Option>, } impl<'a, 'b: 'a> RewriteCheckVisitor<'a, 'b> { @@ -64,7 +60,6 @@ impl<'a, 'b: 'a> RewriteCheckVisitor<'a, 'b> { ) -> ControlFlow> { let caller = CallerInfo { span: target.span(), - source_file: self.source_file.clone(), module: self.module_id, kind, }; diff --git a/assembly/src/assembler/module_graph/mod.rs b/assembly/src/assembler/module_graph/mod.rs index fb5424645e..cf04d34240 100644 --- a/assembly/src/assembler/module_graph/mod.rs +++ b/assembly/src/assembler/module_graph/mod.rs @@ -19,7 +19,7 @@ use crate::ast::InvokeKind; use crate::{ ast::{Export, InvocationTarget, Module, ProcedureIndex, ProcedureName, ResolvedProcedure}, library::{ModuleInfo, ProcedureInfo}, - AssemblyError, LibraryNamespace, LibraryPath, RpoDigest, Spanned, + AssemblyError, LibraryNamespace, LibraryPath, RpoDigest, SourceManager, Spanned, }; // WRAPPER STRUCTS @@ -140,7 +140,7 @@ impl PendingWrappedModule { // MODULE GRAPH // ================================================================================================ -#[derive(Default, Clone)] +#[derive(Clone)] pub struct ModuleGraph { modules: Vec, /// The set of modules pending additional processing before adding them to the graph. @@ -160,11 +160,25 @@ pub struct ModuleGraph { roots: BTreeMap>, kernel_index: Option, kernel: Kernel, + source_manager: Arc, } // ------------------------------------------------------------------------------------------------ /// Constructors impl ModuleGraph { + /// Instantiate a new [ModuleGraph], using the provided [SourceManager] to resolve source info. + pub fn new(source_manager: Arc) -> Self { + Self { + modules: Default::default(), + pending: Default::default(), + callgraph: Default::default(), + roots: Default::default(), + kernel_index: None, + kernel: Default::default(), + source_manager, + } + } + /// Adds all module infos to the graph. pub fn add_compiled_modules( &mut self, @@ -245,13 +259,17 @@ impl ModuleGraph { /// Note: it is assumed that kernel and kernel_module are consistent, but this is not checked. /// /// TODO: consider passing `KerneLibrary` into this constructor as a parameter instead. - pub(super) fn with_kernel(kernel: Kernel, kernel_module: ModuleInfo) -> Self { + pub(super) fn with_kernel( + source_manager: Arc, + kernel: Kernel, + kernel_module: ModuleInfo, + ) -> Self { assert!(!kernel.is_empty()); assert_eq!(kernel_module.path(), &LibraryPath::from(LibraryNamespace::Kernel)); // add kernel module to the graph // TODO: simplify this to avoid using Self::add_compiled_modules() - let mut graph = Self::default(); + let mut graph = Self::new(source_manager); let module_indexes = graph .add_compiled_modules([kernel_module]) .expect("failed to add kernel module to the module graph"); @@ -392,7 +410,6 @@ impl ModuleGraph { if let Export::Alias(ref alias) = procedure { let caller = CallerInfo { span: alias.span(), - source_file: ast_module.source_file(), module: module_id, kind: InvokeKind::ProcRef, }; @@ -408,7 +425,6 @@ impl ModuleGraph { for invoke in procedure.invoked() { let caller = CallerInfo { span: invoke.span(), - source_file: ast_module.source_file(), module: module_id, kind: invoke.kind, }; diff --git a/assembly/src/assembler/module_graph/name_resolver.rs b/assembly/src/assembler/module_graph/name_resolver.rs index 93d325fc72..3904d6fcc3 100644 --- a/assembly/src/assembler/module_graph/name_resolver.rs +++ b/assembly/src/assembler/module_graph/name_resolver.rs @@ -1,4 +1,4 @@ -use alloc::{borrow::Cow, collections::BTreeSet, sync::Arc, vec::Vec}; +use alloc::{borrow::Cow, collections::BTreeSet, vec::Vec}; use super::{ModuleGraph, WrappedModule}; use crate::{ @@ -7,7 +7,7 @@ use crate::{ Ident, InvocationTarget, InvokeKind, Module, ProcedureName, QualifiedProcedureName, ResolvedProcedure, }, - diagnostics::{RelatedLabel, SourceFile}, + diagnostics::RelatedLabel, library::{LibraryNamespace, LibraryPath}, AssemblyError, RpoDigest, SourceSpan, Span, Spanned, }; @@ -21,7 +21,6 @@ use crate::{ /// include in name resolution in order to be able to fully resolve all names for a given set of /// modules. struct ThinModule { - source_file: Option>, path: LibraryPath, resolver: crate::ast::LocalNameResolver, } @@ -34,8 +33,6 @@ struct ThinModule { pub struct CallerInfo { /// The source span of the caller pub span: SourceSpan, - /// The source file corresponding to `span`, if available - pub source_file: Option>, /// The "where", i.e. index of the caller's module in the [ModuleGraph]. pub module: ModuleIndex, /// The "how", i.e. how the callee is being invoked. @@ -114,7 +111,6 @@ impl<'a> NameResolver<'a> { /// have not yet processed to the resolver, as we resolve names for each module in the set. pub fn push_pending(&mut self, module: &Module) { self.pending.push(ThinModule { - source_file: module.source_file(), path: module.path().clone(), resolver: module.resolver(), }); @@ -162,8 +158,8 @@ impl<'a> NameResolver<'a> { }) } None => Err(AssemblyError::UndefinedModule { - span: target.span(), - source_file: caller.source_file.clone(), + span: caller.span, + source_file: self.graph.source_manager.get(caller.span.source_id()).ok(), path: LibraryPath::new_from_components( LibraryNamespace::User(imported_module.clone().into_inner()), [], @@ -229,7 +225,7 @@ impl<'a> NameResolver<'a> { } None => Err(AssemblyError::Failed { labels: vec![RelatedLabel::error("undefined procedure") - .with_source_file(caller.source_file.clone()) + .with_source_file(self.graph.source_manager.get(caller.span.source_id()).ok()) .with_labeled_span(caller.span, "unable to resolve this name locally")], }), } @@ -305,8 +301,12 @@ impl<'a> NameResolver<'a> { loop { let module_index = self.find_module_index(¤t_callee.module).ok_or_else(|| { AssemblyError::UndefinedModule { - span: current_callee.span(), - source_file: current_caller.source_file.clone(), + span: current_caller.span, + source_file: self + .graph + .source_manager + .get(current_caller.span.source_id()) + .ok(), path: current_callee.module.clone(), } })?; @@ -320,8 +320,12 @@ impl<'a> NameResolver<'a> { if matches!(current_caller.kind, InvokeKind::SysCall if self.graph.kernel_index != Some(module_index)) { break Err(AssemblyError::InvalidSysCallTarget { - span: current_callee.span(), - source_file: current_caller.source_file.clone(), + span: current_caller.span, + source_file: self + .graph + .source_manager + .get(current_caller.span.source_id()) + .ok(), callee: current_callee.into_owned(), }); } @@ -335,20 +339,16 @@ impl<'a> NameResolver<'a> { break Err(AssemblyError::Failed { labels: vec![ RelatedLabel::error("recursive alias") - .with_source_file(self.module_source(module_index)) + .with_source_file(self.graph.source_manager.get(fqn.span().source_id()).ok()) .with_labeled_span(fqn.span(), "occurs because this import causes import resolution to loop back on itself"), RelatedLabel::advice("recursive alias") - .with_source_file(caller.source_file.clone()) + .with_source_file(self.graph.source_manager.get(caller.span.source_id()).ok()) .with_labeled_span(caller.span, "as a result of resolving this procedure reference"), ], }); } - let source_file = self - .find_module_index(&fqn.module) - .and_then(|index| self.module_source(index)); current_caller = Cow::Owned(CallerInfo { span: fqn.span(), - source_file, module: module_index, kind: current_caller.kind, }); @@ -363,13 +363,20 @@ impl<'a> NameResolver<'a> { break Err(AssemblyError::Failed { labels: vec![ RelatedLabel::error("undefined procedure") - .with_source_file(caller.source_file.clone()) + .with_source_file( + self.graph.source_manager.get(caller.span.source_id()).ok(), + ) .with_labeled_span( caller.span, "unable to resolve this reference to its definition", ), RelatedLabel::error("name resolution cannot proceed") - .with_source_file(self.module_source(module_index)) + .with_source_file( + self.graph + .source_manager + .get(current_callee.span().source_id()) + .ok(), + ) .with_labeled_span( current_callee.span(), "this name cannot be resolved", @@ -383,10 +390,10 @@ impl<'a> NameResolver<'a> { break Err(AssemblyError::Failed { labels: vec![ RelatedLabel::error("undefined kernel procedure") - .with_source_file(caller.source_file.clone()) + .with_source_file(self.graph.source_manager.get(caller.span.source_id()).ok()) .with_labeled_span(caller.span, "unable to resolve this reference to a procedure in the current kernel"), RelatedLabel::error("invalid syscall") - .with_source_file(self.module_source(module_index)) + .with_source_file(self.graph.source_manager.get(current_callee.span().source_id()).ok()) .with_labeled_span( current_callee.span(), "this name cannot be resolved, because the assembler has an empty kernel", @@ -398,10 +405,10 @@ impl<'a> NameResolver<'a> { break Err(AssemblyError::Failed { labels: vec![ RelatedLabel::error("undefined kernel procedure") - .with_source_file(caller.source_file.clone()) + .with_source_file(self.graph.source_manager.get(caller.span.source_id()).ok()) .with_labeled_span(caller.span, "unable to resolve this reference to a procedure in the current kernel"), RelatedLabel::error("name resolution cannot proceed") - .with_source_file(self.module_source(module_index)) + .with_source_file(self.graph.source_manager.get(current_callee.span().source_id()).ok()) .with_labeled_span( current_callee.span(), "this name cannot be resolved", @@ -415,13 +422,20 @@ impl<'a> NameResolver<'a> { break Err(AssemblyError::Failed { labels: vec![ RelatedLabel::error("undefined procedure") - .with_source_file(caller.source_file.clone()) + .with_source_file( + self.graph.source_manager.get(caller.span.source_id()).ok(), + ) .with_labeled_span( caller.span, "unable to resolve this reference to its definition", ), RelatedLabel::error("name resolution cannot proceed") - .with_source_file(self.module_source(module_index)) + .with_source_file( + self.graph + .source_manager + .get(current_callee.span().source_id()) + .ok(), + ) .with_labeled_span( current_callee.span(), "this name cannot be resolved", @@ -444,19 +458,6 @@ impl<'a> NameResolver<'a> { .map(ModuleIndex::new) } - fn module_source(&self, module: ModuleIndex) -> Option> { - let pending_offset = self.graph.modules.len(); - let module_index = module.as_usize(); - if module_index >= pending_offset { - self.pending[module_index - pending_offset].source_file.clone() - } else { - match &self.graph[module] { - WrappedModule::Ast(module) => module.source_file(), - WrappedModule::Info(_) => None, - } - } - } - fn module_path(&self, module: ModuleIndex) -> LibraryPath { let pending_offset = self.graph.modules.len(); let module_index = module.as_usize(); diff --git a/assembly/src/assembler/module_graph/rewrites/module.rs b/assembly/src/assembler/module_graph/rewrites/module.rs index 7ce8554e38..6d01d32d9c 100644 --- a/assembly/src/assembler/module_graph/rewrites/module.rs +++ b/assembly/src/assembler/module_graph/rewrites/module.rs @@ -1,4 +1,4 @@ -use alloc::{collections::BTreeSet, sync::Arc}; +use alloc::collections::BTreeSet; use core::ops::ControlFlow; use crate::{ @@ -10,8 +10,7 @@ use crate::{ visit::{self, VisitMut}, AliasTarget, InvocationTarget, Invoke, InvokeKind, Module, Procedure, }, - diagnostics::SourceFile, - AssemblyError, Spanned, + AssemblyError, SourceSpan, Spanned, }; // MODULE REWRITE CHECK @@ -26,8 +25,8 @@ use crate::{ pub struct ModuleRewriter<'a, 'b: 'a> { resolver: &'a NameResolver<'b>, module_id: ModuleIndex, + span: SourceSpan, invoked: BTreeSet, - source_file: Option>, } impl<'a, 'b: 'a> ModuleRewriter<'a, 'b> { @@ -36,8 +35,8 @@ impl<'a, 'b: 'a> ModuleRewriter<'a, 'b> { Self { resolver, module_id: ModuleIndex::new(u16::MAX as usize), + span: Default::default(), invoked: Default::default(), - source_file: None, } } @@ -48,7 +47,7 @@ impl<'a, 'b: 'a> ModuleRewriter<'a, 'b> { module: &mut Module, ) -> Result<(), AssemblyError> { self.module_id = module_id; - self.source_file = module.source_file(); + self.span = module.span(); if let ControlFlow::Break(err) = self.visit_mut_module(module) { return Err(err); @@ -64,7 +63,6 @@ impl<'a, 'b: 'a> ModuleRewriter<'a, 'b> { ) -> ControlFlow { let caller = CallerInfo { span: target.span(), - source_file: self.source_file.clone(), module: self.module_id, kind, }; diff --git a/assembly/src/assembler/procedure.rs b/assembly/src/assembler/procedure.rs index b19b05c9fb..e7b98d9d8d 100644 --- a/assembly/src/assembler/procedure.rs +++ b/assembly/src/assembler/procedure.rs @@ -3,8 +3,8 @@ use alloc::{collections::BTreeSet, sync::Arc}; use super::GlobalProcedureIndex; use crate::{ ast::{ProcedureName, QualifiedProcedureName, Visibility}, - diagnostics::SourceFile, - AssemblyError, LibraryPath, RpoDigest, SourceSpan, Spanned, + diagnostics::{SourceManager, SourceSpan, Spanned}, + AssemblyError, LibraryPath, RpoDigest, }; use vm_core::mast::MastNodeId; @@ -15,9 +15,9 @@ pub type CallSet = BTreeSet; /// Information about a procedure currently being compiled. pub struct ProcedureContext { + source_manager: Arc, gid: GlobalProcedureIndex, span: SourceSpan, - source_file: Option>, name: QualifiedProcedureName, visibility: Visibility, num_locals: u16, @@ -31,11 +31,12 @@ impl ProcedureContext { gid: GlobalProcedureIndex, name: QualifiedProcedureName, visibility: Visibility, + source_manager: Arc, ) -> Self { Self { + source_manager, gid, span: name.span(), - source_file: None, name, visibility, num_locals: 0, @@ -52,11 +53,6 @@ impl ProcedureContext { self.span = span; self } - - pub fn with_source_file(mut self, source_file: Option>) -> Self { - self.source_file = source_file; - self - } } // ------------------------------------------------------------------------------------------------ @@ -79,13 +75,14 @@ impl ProcedureContext { &self.name.module } - pub fn source_file(&self) -> Option> { - self.source_file.clone() - } - pub fn is_kernel(&self) -> bool { self.visibility.is_syscall() } + + #[inline(always)] + pub fn source_manager(&self) -> &dyn SourceManager { + self.source_manager.as_ref() + } } // ------------------------------------------------------------------------------------------------ @@ -136,7 +133,6 @@ impl ProcedureContext { pub fn into_procedure(self, mast_root: RpoDigest, mast_node_id: MastNodeId) -> Procedure { Procedure::new(self.name, self.visibility, self.num_locals as u32, mast_root, mast_node_id) .with_span(self.span) - .with_source_file(self.source_file) .with_callset(self.callset) } } @@ -162,7 +158,6 @@ impl Spanned for ProcedureContext { #[derive(Clone, Debug)] pub struct Procedure { span: SourceSpan, - source_file: Option>, path: QualifiedProcedureName, visibility: Visibility, num_locals: u32, @@ -186,7 +181,6 @@ impl Procedure { ) -> Self { Self { span: SourceSpan::default(), - source_file: None, path, visibility, num_locals, @@ -201,11 +195,6 @@ impl Procedure { self } - pub(crate) fn with_source_file(mut self, source_file: Option>) -> Self { - self.source_file = source_file; - self - } - pub(crate) fn with_callset(mut self, callset: CallSet) -> Self { self.callset = callset; self @@ -236,13 +225,6 @@ impl Procedure { &self.path.module } - /// Returns a reference to the Miden Assembly source file from which this - /// procedure was compiled, if available. - #[allow(unused)] - pub fn source_file(&self) -> Option> { - self.source_file.clone() - } - /// Returns the number of memory locals reserved by the procedure. pub fn num_locals(&self) -> u32 { self.num_locals diff --git a/assembly/src/assembler/tests.rs b/assembly/src/assembler/tests.rs index ebde3e94ba..903580d9fd 100644 --- a/assembly/src/assembler/tests.rs +++ b/assembly/src/assembler/tests.rs @@ -1,19 +1,18 @@ -use core::iter; - use pretty_assertions::assert_eq; use vm_core::{assert_matches, mast::MastForest, Program}; use super::{Assembler, Operation}; use crate::{ assembler::{combine_mast_node_ids, mast_forest_builder::MastForestBuilder}, - ast::{Module, ModuleKind}, + diagnostics::Report, + testing::TestContext, }; // TESTS // ================================================================================================ #[test] -fn nested_blocks() { +fn nested_blocks() -> Result<(), Report> { const KERNEL: &str = r#" export.foo add @@ -24,16 +23,17 @@ fn nested_blocks() { push.29 end"#; + let context = TestContext::new(); let assembler = { - let kernel_lib = Assembler::default().assemble_kernel(KERNEL).unwrap(); + let kernel_lib = Assembler::new(context.source_manager()).assemble_kernel(KERNEL).unwrap(); let dummy_module = - Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, MODULE_PROCEDURE) - .unwrap(); - let dummy_library = - Assembler::default().assemble_library(iter::once(dummy_module)).unwrap(); + context.parse_module_with_path(MODULE.parse().unwrap(), MODULE_PROCEDURE)?; + let dummy_library = Assembler::new(context.source_manager()) + .assemble_library([dummy_module]) + .unwrap(); - let mut assembler = Assembler::with_kernel(kernel_lib); + let mut assembler = Assembler::with_kernel(context.source_manager(), kernel_lib); assembler.add_compiled_library(dummy_library).unwrap(); assembler @@ -166,13 +166,15 @@ fn nested_blocks() { // also check that the program has the right number of procedures (which excludes the dummy // library and kernel) assert_eq!(program.num_procedures(), 3); + + Ok(()) } /// Ensures that a single copy of procedures with the same MAST root are added only once to the MAST /// forest. #[test] fn duplicate_procedure() { - let assembler = Assembler::default(); + let context = TestContext::new(); let program_source = r#" proc.foo @@ -192,14 +194,14 @@ fn duplicate_procedure() { end "#; - let program = assembler.assemble_program(program_source).unwrap(); + let program = context.assemble(program_source).unwrap(); assert_eq!(program.num_procedures(), 2); } /// Ensures that equal MAST nodes don't get added twice to a MAST forest #[test] fn duplicate_nodes() { - let assembler = Assembler::default(); + let context = TestContext::new().with_debug_info(false); let program_source = r#" begin @@ -211,7 +213,7 @@ fn duplicate_nodes() { end "#; - let program = assembler.assemble_program(program_source).unwrap(); + let program = context.assemble(program_source).unwrap(); let mut expected_mast_forest = MastForest::new(); @@ -236,7 +238,7 @@ fn duplicate_nodes() { } #[test] -fn explicit_fully_qualified_procedure_references() { +fn explicit_fully_qualified_procedure_references() -> Result<(), Report> { const BAR_NAME: &str = "foo::bar"; const BAR: &str = r#" export.bar @@ -248,11 +250,14 @@ fn explicit_fully_qualified_procedure_references() { exec.::foo::bar::bar end"#; - let bar = Module::parse_str(BAR_NAME.parse().unwrap(), ModuleKind::Library, BAR).unwrap(); - let baz = Module::parse_str(BAZ_NAME.parse().unwrap(), ModuleKind::Library, BAZ).unwrap(); - let library = Assembler::default().assemble_library([bar, baz]).unwrap(); + let context = TestContext::default(); + let bar = context.parse_module_with_path(BAR_NAME.parse().unwrap(), BAR)?; + let baz = context.parse_module_with_path(BAZ_NAME.parse().unwrap(), BAZ)?; + let library = context.assemble_library([bar, baz]).unwrap(); - let assembler = Assembler::default().with_compiled_library(&library).unwrap(); + let assembler = Assembler::new(context.source_manager()) + .with_compiled_library(&library) + .unwrap(); let program = r#" begin @@ -260,10 +265,11 @@ fn explicit_fully_qualified_procedure_references() { end"#; assert_matches!(assembler.assemble_program(program), Ok(_)); + Ok(()) } #[test] -fn re_exports() { +fn re_exports() -> Result<(), Report> { const BAR_NAME: &str = "foo::bar"; const BAR: &str = r#" export.bar @@ -280,11 +286,14 @@ fn re_exports() { push.1 push.2 add end"#; - let bar = Module::parse_str(BAR_NAME.parse().unwrap(), ModuleKind::Library, BAR).unwrap(); - let baz = Module::parse_str(BAZ_NAME.parse().unwrap(), ModuleKind::Library, BAZ).unwrap(); - let library = Assembler::default().assemble_library([bar, baz]).unwrap(); + let context = TestContext::new(); + let bar = context.parse_module_with_path(BAR_NAME.parse().unwrap(), BAR)?; + let baz = context.parse_module_with_path(BAZ_NAME.parse().unwrap(), BAZ)?; + let library = context.assemble_library([bar, baz]).unwrap(); - let assembler = Assembler::default().with_compiled_library(&library).unwrap(); + let assembler = Assembler::new(context.source_manager()) + .with_compiled_library(&library) + .unwrap(); let program = r#" use.foo::baz @@ -297,4 +306,5 @@ fn re_exports() { end"#; assert_matches!(assembler.assemble_program(program), Ok(_)); + Ok(()) } diff --git a/assembly/src/ast/invocation_target.rs b/assembly/src/ast/invocation_target.rs index 52c85ca83b..01d3984c63 100644 --- a/assembly/src/ast/invocation_target.rs +++ b/assembly/src/ast/invocation_target.rs @@ -122,7 +122,7 @@ impl InvocationTarget { pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { target.write_u8(self.tag()); match self { - Self::MastRoot(spanned) => spanned.write_into(target, options), + Self::MastRoot(spanned) => spanned.write_into_with_options(target, options.debug_info), Self::ProcedureName(name) => name.write_into_with_options(target, options), Self::ProcedurePath { name, module } => { name.write_into_with_options(target, options); @@ -142,7 +142,7 @@ impl InvocationTarget { ) -> Result { match source.read_u8()? { 0 => { - let root = Span::::read_from(source, options)?; + let root = Span::::read_from_with_options(source, options.debug_info)?; Ok(Self::MastRoot(root)) } 1 => { diff --git a/assembly/src/ast/module.rs b/assembly/src/ast/module.rs index b898228a87..fcfb38b53c 100644 --- a/assembly/src/ast/module.rs +++ b/assembly/src/ast/module.rs @@ -1,9 +1,4 @@ -use alloc::{ - boxed::Box, - string::{String, ToString}, - sync::Arc, - vec::Vec, -}; +use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use core::fmt; use super::{ @@ -112,12 +107,6 @@ impl Deserializable for ModuleKind { pub struct Module { /// The span covering the entire definition of this module. span: SourceSpan, - /// If available/known, the source contents from which this module was parsed. This is used - /// to provide rich diagnostics output during semantic analysis. - /// - /// In cases where this file is not available, diagnostics will revert to a simple form with - /// a helpful message, but without source code snippets. - source_file: Option>, /// The documentation associated with this module. /// /// Module documentation is provided in Miden Assembly as a documentation comment starting on @@ -144,7 +133,6 @@ impl Module { pub fn new(kind: ModuleKind, path: LibraryPath) -> Self { Self { span: Default::default(), - source_file: None, docs: None, path, kind, @@ -163,14 +151,6 @@ impl Module { Self::new(ModuleKind::Executable, LibraryNamespace::Exec.into()) } - /// Builds this [Module] with the given source file in which it was defined. - /// - /// When a source file is given, diagnostics will contain source code snippets. - pub fn with_source_file(mut self, source_file: Option>) -> Self { - self.source_file = source_file; - self - } - /// Specifies the source span in the source file in which this module was defined, that covers /// the full definition of this module. pub fn with_span(mut self, span: SourceSpan) -> Self { @@ -178,11 +158,6 @@ impl Module { self } - /// Like [Module::with_source_file], but does not require ownership of the [Module]. - pub fn set_source_file(&mut self, source_file: Arc) { - self.source_file = Some(source_file); - } - /// Sets the [LibraryPath] for this module pub fn set_path(&mut self, path: LibraryPath) { self.path = path; @@ -250,26 +225,6 @@ impl Module { /// Parsing impl Module { - /// Parse a [Module], `name`, of the given [ModuleKind], from `path`. - #[cfg(feature = "std")] - pub fn parse_file

(name: LibraryPath, kind: ModuleKind, path: P) -> Result, Report> - where - P: AsRef, - { - let mut parser = Self::parser(kind); - parser.parse_file(name, path) - } - - /// Parse a [Module], `name`, of the given [ModuleKind], from `source`. - pub fn parse_str( - name: LibraryPath, - kind: ModuleKind, - source: impl ToString, - ) -> Result, Report> { - let mut parser = Self::parser(kind); - parser.parse_str(name, source) - } - /// Parse a [Module], `name`, of the given [ModuleKind], from `source_file`. pub fn parse( name: LibraryPath, @@ -281,10 +236,6 @@ impl Module { } /// Get a [ModuleParser] for parsing modules of the provided [ModuleKind] - /// - /// This is mostly useful when you want tighter control over the parser configuration, otherwise - /// it is generally more convenient to use [Module::parse_file] or [Module::parse_str] for most - /// use cases. pub fn parser(kind: ModuleKind) -> ModuleParser { ModuleParser::new(kind) } @@ -292,17 +243,6 @@ impl Module { /// Metadata impl Module { - /// Get the source code for this module, if available - /// - /// The source code will not be available in the following situations: - /// - /// * The module was constructed in-memory via AST structures, and not derived from source code. - /// * The module was serialized without debug info, and then deserialized. Without debug info, - /// the source code is lost when round-tripping through serialization. - pub fn source_file(&self) -> Option> { - self.source_file.clone() - } - /// Get the name of this specific module, i.e. the last component of the [LibraryPath] that /// represents the fully-qualified name of the module, e.g. `u64` in `std::math::u64` pub fn name(&self) -> &str { @@ -533,17 +473,6 @@ impl Module { options.write_into(target); if options.debug_info { self.span.write_into(target); - if let Some(source_file) = self.source_file.as_ref() { - target.write_u8(1); - let source_name = source_file.name(); - let source_bytes = source_file.inner().as_bytes(); - target.write_usize(source_name.as_bytes().len()); - target.write_bytes(source_name.as_bytes()); - target.write_usize(source_bytes.len()); - target.write_bytes(source_bytes); - } else { - target.write_u8(0); - } } self.kind.write_into(target); self.path.write_into(target); @@ -627,30 +556,10 @@ impl Deserializable for Module { /// [Module::write_into_with_options] fn read_from(source: &mut R) -> Result { let options = AstSerdeOptions::read_from(source)?; - let (span, source_file) = if options.debug_info { - let span = SourceSpan::read_from(source)?; - match source.read_u8()? { - 0 => (span, None), - 1 => { - let nlen = source.read_usize()?; - let source_name = core::str::from_utf8(source.read_slice(nlen)?) - .map(|s| s.to_string()) - .map_err(|e| DeserializationError::InvalidValue(e.to_string()))?; - let clen = source.read_usize()?; - let source_content = core::str::from_utf8(source.read_slice(clen)?) - .map_err(|e| DeserializationError::InvalidValue(e.to_string()))?; - let source_file = - Arc::new(SourceFile::new(source_name, source_content.to_string())); - (span, Some(source_file)) - } - n => { - return Err(DeserializationError::InvalidValue(format!( - "invalid option tag: '{n}'" - ))); - } - } + let span = if options.debug_info { + SourceSpan::read_from(source)? } else { - (SourceSpan::default(), None) + SourceSpan::default() }; let kind = ModuleKind::read_from(source)?; let path = LibraryPath::read_from(source)?; @@ -669,11 +578,10 @@ impl Deserializable for Module { let mut procedures = Vec::with_capacity(num_procedures); for _ in 0..num_procedures { let export = Export::read_from_with_options(source, options)?; - procedures.push(export.with_source_file(source_file.clone())); + procedures.push(export); } Ok(Self { span, - source_file, docs: None, path, kind, diff --git a/assembly/src/ast/procedure/alias.rs b/assembly/src/ast/procedure/alias.rs index 1a7b58ce0f..fe9e442160 100644 --- a/assembly/src/ast/procedure/alias.rs +++ b/assembly/src/ast/procedure/alias.rs @@ -1,11 +1,11 @@ -use alloc::{string::String, sync::Arc}; +use alloc::string::String; use core::fmt; use super::{ProcedureName, QualifiedProcedureName}; use crate::{ ast::{AstSerdeOptions, InvocationTarget}, - diagnostics::SourceFile, - ByteReader, ByteWriter, DeserializationError, RpoDigest, SourceSpan, Span, Spanned, + diagnostics::{SourceSpan, Span, Spanned}, + ByteReader, ByteWriter, DeserializationError, RpoDigest, }; // PROCEDURE ALIAS @@ -19,8 +19,6 @@ use crate::{ /// the caller is in the current module, or in another module. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProcedureAlias { - /// The source file in which this alias was defined, if available - source_file: Option>, /// The documentation attached to this procedure docs: Option>, /// The name of this procedure @@ -38,7 +36,6 @@ impl ProcedureAlias { pub fn new(name: ProcedureName, target: AliasTarget) -> Self { Self { docs: None, - source_file: None, name, target, } @@ -50,17 +47,6 @@ impl ProcedureAlias { self } - /// Adds source code to this declaration, so that we can render source snippets in diagnostics. - pub fn with_source_file(mut self, source_file: Option>) -> Self { - self.source_file = source_file; - self - } - - /// Returns the source file associated with this declaration. - pub fn source_file(&self) -> Option> { - self.source_file.clone() - } - /// Returns the documentation associated with this declaration. pub fn docs(&self) -> Option<&Span> { self.docs.as_ref() @@ -119,7 +105,6 @@ impl ProcedureAlias { let name = ProcedureName::read_from_with_options(source, options)?; let target = AliasTarget::read_from_with_options(source, options)?; Ok(Self { - source_file: None, docs: None, name, target, @@ -283,7 +268,7 @@ impl AliasTarget { pub fn write_into_with_options(&self, target: &mut W, options: AstSerdeOptions) { target.write_u8(self.tag()); match self { - Self::MastRoot(spanned) => spanned.write_into(target, options), + Self::MastRoot(spanned) => spanned.write_into_with_options(target, options.debug_info), Self::ProcedurePath(path) => path.write_into_with_options(target, options), Self::AbsoluteProcedurePath(path) => path.write_into_with_options(target, options), } @@ -296,7 +281,7 @@ impl AliasTarget { ) -> Result { match source.read_u8()? { 0 => { - let root = Span::::read_from(source, options)?; + let root = Span::::read_from_with_options(source, options.debug_info)?; Ok(Self::MastRoot(root)) } 1 => { diff --git a/assembly/src/ast/procedure/mod.rs b/assembly/src/ast/procedure/mod.rs index b6c03730d2..72fc0033ec 100644 --- a/assembly/src/ast/procedure/mod.rs +++ b/assembly/src/ast/procedure/mod.rs @@ -13,10 +13,9 @@ pub use self::resolver::{LocalNameResolver, ResolvedProcedure}; use crate::{ ast::{AstSerdeOptions, Invoke}, - diagnostics::SourceFile, ByteReader, ByteWriter, DeserializationError, SourceSpan, Span, Spanned, }; -use alloc::{string::String, sync::Arc}; +use alloc::string::String; // EXPORT // ================================================================================================ @@ -43,23 +42,6 @@ impl Export { } } - /// Adds the source file in which this export was defined, which will allow diagnostics to - /// contain source snippets when emitted. - pub fn with_source_file(self, source_file: Option>) -> Self { - match self { - Self::Procedure(proc) => Self::Procedure(proc.with_source_file(source_file)), - Self::Alias(alias) => Self::Alias(alias.with_source_file(source_file)), - } - } - - /// Returns the source file in which this export was defined. - pub fn source_file(&self) -> Option> { - match self { - Self::Procedure(ref proc) => proc.source_file(), - Self::Alias(ref alias) => alias.source_file(), - } - } - /// Returns the name of the exported procedure. pub fn name(&self) -> &ProcedureName { match self { diff --git a/assembly/src/ast/procedure/procedure.rs b/assembly/src/ast/procedure/procedure.rs index fe31506977..3fa072384d 100644 --- a/assembly/src/ast/procedure/procedure.rs +++ b/assembly/src/ast/procedure/procedure.rs @@ -1,10 +1,9 @@ -use alloc::{collections::BTreeSet, string::String, sync::Arc}; +use alloc::{collections::BTreeSet, string::String}; use core::fmt; use super::ProcedureName; use crate::{ ast::{AstSerdeOptions, Block, Invoke}, - diagnostics::SourceFile, ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable, SourceSpan, Span, Spanned, }; @@ -72,8 +71,6 @@ impl Deserializable for Visibility { pub struct Procedure { /// The source span of the full procedure body span: SourceSpan, - /// The source file in which this procedure was defined, if available - source_file: Option>, /// The documentation attached to this procedure docs: Option>, /// The local name of this procedure @@ -101,7 +98,6 @@ impl Procedure { ) -> Self { Self { span, - source_file: None, docs: None, name, visibility, @@ -117,18 +113,6 @@ impl Procedure { self } - /// Adds source code to this procedure definition so we can render source snippets - /// in diagnostics. - pub fn with_source_file(mut self, source_file: Option>) -> Self { - self.source_file = source_file; - self - } - - /// Like [Procedure::with_source_file], but does not require ownership of the procedure. - pub fn set_source_file(&mut self, source_file: Arc) { - self.source_file = Some(source_file); - } - /// Modifies the visibility of this procedure. /// /// This is made crate-local as the visibility of a procedure is virtually always determined @@ -142,11 +126,6 @@ impl Procedure { /// Metadata impl Procedure { - /// Returns the source file associated with this procedure. - pub fn source_file(&self) -> Option> { - self.source_file.clone() - } - /// Returns the name of this procedure within its containing module. pub fn name(&self) -> &ProcedureName { &self.name @@ -264,7 +243,6 @@ impl Procedure { let body = Block::read_from_with_options(source, options)?; Ok(Self { span, - source_file: None, docs: None, name, visibility, diff --git a/assembly/src/ast/tests.rs b/assembly/src/ast/tests.rs index d82fb240cb..ba443e477f 100644 --- a/assembly/src/ast/tests.rs +++ b/assembly/src/ast/tests.rs @@ -288,9 +288,9 @@ macro_rules! assert_program_diagnostic_lines { /// Tests the AST parsing #[test] fn test_ast_parsing_program_simple() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); - let source = source_file!("begin push.0 assertz add.1 end"); + let source = source_file!(&context, "begin push.0 assertz add.1 end"); let forms = module!(begin!(inst!(PushU8(0)), inst!(Assertz), inst!(Incr))); assert_eq!(context.parse_forms(source)?, forms); @@ -300,9 +300,10 @@ fn test_ast_parsing_program_simple() -> Result<(), Report> { #[test] fn test_ast_parsing_program_push() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" begin push.10 push.500 push.70000 push.5000000000 @@ -338,11 +339,11 @@ fn test_ast_parsing_program_push() -> Result<(), Report> { assert_eq!(context.parse_forms(source)?, forms); // Push a hexadecimal string containing more than 4 values - let source_too_long = source_file!("begin push.0x00000000000000001000000000000000200000000000000030000000000000004000000000000000"); + let source_too_long = source_file!(&context, "begin push.0x00000000000000001000000000000000200000000000000030000000000000004000000000000000"); assert_parse_diagnostic!(source_too_long, "long hex strings must contain exactly 64 digits"); // Push a hexadecimal string containing less than 4 values - let source_too_long = source_file!("begin push.0x00000000000000001000000000000000"); + let source_too_long = source_file!(&context, "begin push.0x00000000000000001000000000000000"); assert_parse_diagnostic!(source_too_long, "expected 2, 4, 8, 16, or 64 hex digits"); Ok(()) @@ -350,9 +351,10 @@ fn test_ast_parsing_program_push() -> Result<(), Report> { #[test] fn test_ast_parsing_program_u32() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" begin push.3 @@ -385,9 +387,10 @@ fn test_ast_parsing_program_u32() -> Result<(), Report> { #[test] fn test_ast_parsing_program_proc() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" proc.foo.1 loc_load.0 @@ -413,8 +416,9 @@ fn test_ast_parsing_program_proc() -> Result<(), Report> { #[test] fn test_ast_parsing_module() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" export.foo.1 loc_load.0 @@ -427,8 +431,8 @@ fn test_ast_parsing_module() -> Result<(), Report> { #[test] fn test_ast_parsing_adv_ops() -> Result<(), Report> { - let mut context = TestContext::new(); - let source = source_file!("begin adv_push.1 adv_loadw end"); + let context = TestContext::new(); + let source = source_file!(&context, "begin adv_push.1 adv_loadw end"); let forms = module!(begin!(inst!(AdvPush(1u8.into())), inst!(AdvLoadW))); assert_eq!(context.parse_forms(source)?, forms); Ok(()) @@ -438,9 +442,11 @@ fn test_ast_parsing_adv_ops() -> Result<(), Report> { fn test_ast_parsing_adv_injection() -> Result<(), Report> { use super::AdviceInjectorNode::*; - let mut context = TestContext::new(); - let source = - source_file!("begin adv.push_u64div adv.push_mapval adv.push_smtget adv.insert_mem end"); + let context = TestContext::new(); + let source = source_file!( + &context, + "begin adv.push_u64div adv.push_mapval adv.push_smtget adv.insert_mem end" + ); let forms = module!(begin!( inst!(AdvInject(PushU64Div)), inst!(AdvInject(PushMapVal)), @@ -453,8 +459,8 @@ fn test_ast_parsing_adv_injection() -> Result<(), Report> { #[test] fn test_ast_parsing_bitwise_counters() -> Result<(), Report> { - let mut context = TestContext::new(); - let source = source_file!("begin u32clz u32ctz u32clo u32cto end"); + let context = TestContext::new(); + let source = source_file!(&context, "begin u32clz u32ctz u32clo u32cto end"); let forms = module!(begin!(inst!(U32Clz), inst!(U32Ctz), inst!(U32Clo), inst!(U32Cto))); assert_eq!(context.parse_forms(source)?, forms); @@ -463,8 +469,8 @@ fn test_ast_parsing_bitwise_counters() -> Result<(), Report> { #[test] fn test_ast_parsing_ilog2() -> Result<(), Report> { - let mut context = TestContext::new(); - let source = source_file!("begin push.8 ilog2 end"); + let context = TestContext::new(); + let source = source_file!(&context, "begin push.8 ilog2 end"); let forms = module!(begin!(inst!(PushU8(8)), inst!(ILog2))); assert_eq!(context.parse_forms(source)?, forms); @@ -473,8 +479,9 @@ fn test_ast_parsing_ilog2() -> Result<(), Report> { #[test] fn test_ast_parsing_use() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" use.std::abc::foo begin @@ -489,8 +496,9 @@ fn test_ast_parsing_use() -> Result<(), Report> { #[test] fn test_ast_parsing_module_nested_if() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" proc.foo push.1 @@ -532,8 +540,9 @@ fn test_ast_parsing_module_nested_if() -> Result<(), Report> { #[test] fn test_ast_parsing_module_sequential_if() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" proc.foo push.1 @@ -567,8 +576,9 @@ fn test_ast_parsing_module_sequential_if() -> Result<(), Report> { #[test] fn parsed_while_if_body() { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, "\ begin push.1 @@ -600,8 +610,9 @@ fn parsed_while_if_body() { #[test] fn test_missing_import() { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" begin exec.u64::add @@ -629,7 +640,9 @@ fn test_missing_import() { #[test] fn test_use_in_proc_body() { + let context = TestContext::default(); let source = source_file!( + &context, r#" export.foo.1 loc_load.0 @@ -653,7 +666,8 @@ fn test_use_in_proc_body() { #[test] fn test_unterminated_proc() { - let source = source_file!("proc.foo add mul begin push.1 end"); + let context = TestContext::default(); + let source = source_file!(&context, "proc.foo add mul begin push.1 end"); assert_parse_diagnostic_lines!( source, @@ -669,7 +683,8 @@ fn test_unterminated_proc() { #[test] fn test_unterminated_if() { - let source = source_file!("proc.foo add mul if.true add.2 begin push.1 end"); + let context = TestContext::default(); + let source = source_file!(&context, "proc.foo add mul if.true add.2 begin push.1 end"); assert_parse_diagnostic_lines!( source, @@ -688,8 +703,9 @@ fn test_unterminated_if() { #[test] fn test_ast_parsing_simple_docs() -> Result<(), Report> { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, r#" #! proc doc export.foo.1 @@ -704,9 +720,10 @@ fn test_ast_parsing_simple_docs() -> Result<(), Report> { #[test] fn test_ast_parsing_module_docs_valid() { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, "\ #! Test documentation for the whole module in parsing test. Lorem ipsum dolor sit amet, #! consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. @@ -783,8 +800,9 @@ end" #[test] fn test_ast_parsing_module_docs_fail() { - let mut context = TestContext::new(); + let context = TestContext::new(); let source = source_file!( + &context, "\ #! module doc @@ -813,6 +831,7 @@ fn test_ast_parsing_module_docs_fail() { ); let source = source_file!( + &context, "\ #! proc doc export.foo.1 @@ -839,6 +858,7 @@ fn test_ast_parsing_module_docs_fail() { ); let source = source_file!( + &context, "\ #! module doc @@ -862,6 +882,7 @@ fn test_ast_parsing_module_docs_fail() { ); let source = source_file!( + &context, "\ export.foo.1 loc_load.0 @@ -887,6 +908,7 @@ fn test_ast_parsing_module_docs_fail() { ); let source = source_file!( + &context, "\ #! module doc @@ -914,6 +936,7 @@ fn test_ast_parsing_module_docs_fail() { ); let source = source_file!( + &context, "\ #! proc doc export.foo.1 @@ -943,7 +966,9 @@ fn test_ast_parsing_module_docs_fail() { #[test] fn assert_parsing_line_unmatched_begin() { + let context = TestContext::default(); let source = source_file!( + &context, "\ begin push.1.2 @@ -964,7 +989,9 @@ fn assert_parsing_line_unmatched_begin() { #[test] fn assert_parsing_line_extra_param() { + let context = TestContext::default(); let source = source_file!( + &context, "\ begin add.1.2 @@ -986,7 +1013,9 @@ fn assert_parsing_line_extra_param() { #[test] fn assert_parsing_line_invalid_op() { + let context = TestContext::default(); let source = source_file!( + &context, "\ begin repeat.3 @@ -1036,7 +1065,9 @@ fn assert_parsing_line_invalid_op() { #[test] fn assert_parsing_line_unexpected_token() { + let context = TestContext::default(); let source = source_file!( + &context, "\ proc.foo add diff --git a/assembly/src/compile.rs b/assembly/src/compile.rs index 359ee1bfb4..9511e31ce8 100644 --- a/assembly/src/compile.rs +++ b/assembly/src/compile.rs @@ -8,7 +8,10 @@ use alloc::{ use crate::{ ast::{Module, ModuleKind}, - diagnostics::{IntoDiagnostic, NamedSource, Report, SourceCode, SourceFile, WrapErr}, + diagnostics::{ + IntoDiagnostic, NamedSource, Report, SourceCode, SourceContent, SourceFile, SourceManager, + WrapErr, + }, library::{LibraryNamespace, LibraryPath}, }; @@ -93,8 +96,8 @@ pub trait Compile: Sized { /// /// See [Compile::compile_with_options()] for more details. #[inline] - fn compile(self) -> Result, Report> { - self.compile_with_options(Options::default()) + fn compile(self, source_manager: &dyn SourceManager) -> Result, Report> { + self.compile_with_options(source_manager, Options::default()) } /// Compile (or convert) `self` into a [Module] using the provided `options`. @@ -104,7 +107,11 @@ pub trait Compile: Sized { /// an executable module). /// /// See the documentation for [Options] to see how compilation can be configured. - fn compile_with_options(self, options: Options) -> Result, Report>; + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report>; } // COMPILE IMPLEMENTATIONS FOR MODULES @@ -112,20 +119,32 @@ pub trait Compile: Sized { impl Compile for Module { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - Box::new(self).compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + Box::new(self).compile_with_options(source_manager, options) } } impl<'a> Compile for &'a Module { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - Box::new(self.clone()).compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + Box::new(self.clone()).compile_with_options(source_manager, options) } } impl Compile for Box { - fn compile_with_options(mut self, options: Options) -> Result, Report> { + fn compile_with_options( + mut self, + _source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { let actual = self.kind(); if actual == options.kind { if let Some(path) = options.path { @@ -143,8 +162,12 @@ impl Compile for Box { impl Compile for Arc { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - Box::new(Arc::unwrap_or_clone(self)).compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + Box::new(Arc::unwrap_or_clone(self)).compile_with_options(source_manager, options) } } @@ -152,10 +175,15 @@ impl Compile for Arc { // ------------------------------------------------------------------------------------------------ impl Compile for Arc { - fn compile_with_options(self, options: Options) -> Result, Report> { + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + let source_file = source_manager.copy_into(&self); let path = match options.path { Some(path) => path, - None => self + None => source_file .name() .parse::() .into_diagnostic() @@ -163,46 +191,72 @@ impl Compile for Arc { }; let mut parser = Module::parser(options.kind); parser.set_warnings_as_errors(options.warnings_as_errors); - parser.parse(path, self) + parser.parse(path, source_file) } } impl<'a> Compile for &'a str { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - self.to_string().compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + self.to_string().into_boxed_str().compile_with_options(source_manager, options) } } impl<'a> Compile for &'a String { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - self.clone().compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + self.clone().into_boxed_str().compile_with_options(source_manager, options) } } impl Compile for String { - fn compile_with_options(self, options: Options) -> Result, Report> { + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + self.into_boxed_str().compile_with_options(source_manager, options) + } +} + +impl Compile for Box { + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + let path = options.path.unwrap_or_else(|| { + LibraryPath::from(match options.kind { + ModuleKind::Library => LibraryNamespace::Anon, + ModuleKind::Executable => LibraryNamespace::Exec, + ModuleKind::Kernel => LibraryNamespace::Kernel, + }) + }); + let name = Arc::::from(path.path().into_owned().into_boxed_str()); let mut parser = Module::parser(options.kind); parser.set_warnings_as_errors(options.warnings_as_errors); - if let Some(path) = options.path { - let source = Arc::new(SourceFile::new(path.path(), self)); - return parser.parse(path, source); - } - let path = LibraryPath::from(match options.kind { - ModuleKind::Library => LibraryNamespace::Anon, - ModuleKind::Executable => LibraryNamespace::Exec, - ModuleKind::Kernel => LibraryNamespace::Kernel, - }); - let source = Arc::new(SourceFile::new(path.path(), self)); - parser.parse(path, source) + let content = SourceContent::new(name.clone(), self); + let source_file = source_manager.load_from_raw_parts(name, content); + parser.parse(path, source_file) } } impl<'a> Compile for Cow<'a, str> { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - self.into_owned().compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + self.into_owned().into_boxed_str().compile_with_options(source_manager, options) } } @@ -210,27 +264,51 @@ impl<'a> Compile for Cow<'a, str> { // ------------------------------------------------------------------------------------------------ impl<'a> Compile for &'a [u8] { - #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { + #[inline] + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { core::str::from_utf8(self) .map_err(|err| { - Report::from(crate::parser::ParsingError::from(err)).with_source_code(self.to_vec()) + Report::from(crate::parser::ParsingError::from_utf8_error(Default::default(), err)) + .with_source_code(self.to_vec()) }) .wrap_err("parsing failed: invalid source code") - .and_then(|source| source.compile_with_options(options)) + .and_then(|source| source.compile_with_options(source_manager, options)) } } impl Compile for Vec { - #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { + #[inline] + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { String::from_utf8(self) .map_err(|err| { - let error = crate::parser::ParsingError::from(err.utf8_error()); + let error = crate::parser::ParsingError::from_utf8_error( + Default::default(), + err.utf8_error(), + ); Report::from(error).with_source_code(err.into_bytes()) }) .wrap_err("parsing failed: invalid source code") - .and_then(|source| source.compile_with_options(options)) + .and_then(|source| { + source.into_boxed_str().compile_with_options(source_manager, options) + }) + } +} +impl Compile for Box<[u8]> { + #[inline(always)] + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + Vec::from(self).compile_with_options(source_manager, options) } } @@ -238,14 +316,31 @@ impl Compile for NamedSource where T: SourceCode + AsRef<[u8]>, { - fn compile_with_options(self, options: Options) -> Result, Report> { - let content = String::from_utf8(self.inner().as_ref().to_vec()) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + let path = match options.path { + Some(path) => path, + None => self + .name() + .parse::() + .into_diagnostic() + .wrap_err("cannot compile module as it has an invalid path/name")?, + }; + let content = core::str::from_utf8(self.inner().as_ref()) .map_err(|err| { - let error = crate::parser::ParsingError::from(err.utf8_error()); - Report::from(error).with_source_code(err.into_bytes()) + let error = crate::parser::ParsingError::from_utf8_error(Default::default(), err); + Report::from(error) }) .wrap_err("parsing failed: expected source code to be valid utf-8")?; - Arc::new(SourceFile::new(self.name(), content)).compile_with_options(options) + let name = Arc::::from(self.name()); + let content = SourceContent::new(name.clone(), content.to_string().into_boxed_str()); + let source_file = source_manager.load_from_raw_parts(name, content); + let mut parser = Module::parser(options.kind); + parser.set_warnings_as_errors(options.warnings_as_errors); + parser.parse(path, source_file) } } @@ -254,9 +349,14 @@ where #[cfg(feature = "std")] impl<'a> Compile for &'a std::path::Path { - fn compile_with_options(self, options: Options) -> Result, Report> { + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { use crate::{ast::Ident, library::PathError}; use std::path::Component; + use vm_core::debuginfo::SourceManagerExt; let path = match options.path { Some(path) => path, @@ -292,15 +392,23 @@ impl<'a> Compile for &'a std::path::Path { LibraryPath::new_from_components(ns, parts) } }; + let source_file = source_manager + .load_file(self) + .into_diagnostic() + .wrap_err("source manager is unable to load file")?; let mut parser = Module::parser(options.kind); - parser.parse_file(path, self) + parser.parse(path, source_file) } } #[cfg(feature = "std")] impl Compile for std::path::PathBuf { #[inline(always)] - fn compile_with_options(self, options: Options) -> Result, Report> { - self.as_path().compile_with_options(options) + fn compile_with_options( + self, + source_manager: &dyn SourceManager, + options: Options, + ) -> Result, Report> { + self.as_path().compile_with_options(source_manager, options) } } diff --git a/assembly/src/diagnostics.rs b/assembly/src/diagnostics.rs index 09e049bc5f..0acfaa68dd 100644 --- a/assembly/src/diagnostics.rs +++ b/assembly/src/diagnostics.rs @@ -7,7 +7,7 @@ pub use tracing; use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec::Vec}; use core::{fmt, ops::Range}; -pub type SourceFile = NamedSource; +pub use vm_core::debuginfo::*; // LABEL // ================================================================================================ @@ -166,7 +166,7 @@ impl RelatedLabel { self } - pub fn with_labeled_span(self, span: super::SourceSpan, message: S) -> Self + pub fn with_labeled_span(self, span: SourceSpan, message: S) -> Self where Cow<'static, str>: From, { diff --git a/assembly/src/lib.rs b/assembly/src/lib.rs index 09c4075ffa..dab2ca205d 100644 --- a/assembly/src/lib.rs +++ b/assembly/src/lib.rs @@ -32,9 +32,12 @@ mod tests; pub use self::assembler::Assembler; pub use self::compile::{Compile, Options as CompileOptions}; +pub use self::diagnostics::{ + DefaultSourceManager, Report, SourceFile, SourceId, SourceManager, SourceSpan, Span, Spanned, +}; pub use self::errors::{AssemblyError, CompiledLibraryError}; pub use self::library::{LibraryError, LibraryNamespace, LibraryPath, PathError, Version}; -pub use self::parser::{ModuleParser, SourceLocation, SourceSpan, Span, Spanned}; +pub use self::parser::ModuleParser; /// Re-exported for downstream crates pub use vm_core::utils; diff --git a/assembly/src/library/mod.rs b/assembly/src/library/mod.rs index 32cc3e882f..ede6a4d61c 100644 --- a/assembly/src/library/mod.rs +++ b/assembly/src/library/mod.rs @@ -219,13 +219,12 @@ mod use_std_library { use super::*; use crate::{ ast::{self, ModuleKind}, - diagnostics::IntoDiagnostic, + diagnostics::{IntoDiagnostic, SourceManager}, Assembler, }; - use alloc::collections::btree_map::Entry; use masl::{LibraryEntry, WalkLibrary}; use miette::{Context, Report}; - use std::{fs, io, path::Path}; + use std::{collections::btree_map::Entry, fs, io, path::Path, sync::Arc}; use vm_core::utils::ReadAdapter; impl CompiledLibrary { @@ -285,7 +284,11 @@ mod use_std_library { /// For example, let's say I call this function like so: /// /// ```rust - /// CompiledLibrary::from_dir("~/masm/std", LibraryNamespace::new("std").unwrap()): + /// CompiledLibrary::from_dir( + /// "~/masm/std", + /// LibraryNamespace::new("std").unwrap() + /// Arc::new(crate::DefaultSourceManager::default()), + /// ); /// ``` /// /// Here's how we would handle various files under this path: @@ -298,6 +301,7 @@ mod use_std_library { pub fn from_dir( path: impl AsRef, namespace: LibraryNamespace, + source_manager: Arc, ) -> Result { let path = path.as_ref(); if !path.is_dir() { @@ -312,7 +316,7 @@ mod use_std_library { return Err(Report::msg("mod.masm is not allowed in the root directory")); } - Self::compile_modules_from_dir(namespace, path) + Self::compile_modules_from_dir(namespace, path, source_manager) } /// Read the contents (modules) of this library from `dir`, returning any errors that occur @@ -325,6 +329,7 @@ mod use_std_library { fn compile_modules_from_dir( namespace: LibraryNamespace, dir: &Path, + source_manager: Arc, ) -> Result { let mut modules = BTreeMap::default(); @@ -340,7 +345,8 @@ mod use_std_library { name.pop(); } // Parse module at the given path - let ast = ast::Module::parse_file(name.clone(), ModuleKind::Library, &source_path)?; + let mut parser = ast::Module::parser(ModuleKind::Library); + let ast = parser.parse_file(name.clone(), &source_path, &source_manager)?; match modules.entry(name) { Entry::Occupied(ref entry) => { return Err(LibraryError::DuplicateModulePath(entry.key().clone())) @@ -364,7 +370,9 @@ mod use_std_library { .into()); } - Assembler::default().assemble_library(modules.into_values()) + Assembler::new(source_manager) + .with_debug_mode(true) + .assemble_library(modules.into_values()) } pub fn deserialize_from_file(path: impl AsRef) -> Result { @@ -529,9 +537,10 @@ impl Deserializable for KernelLibrary { #[cfg(feature = "std")] mod use_std_kernel { + use std::{io, path::Path, sync::Arc}; + use super::*; - use miette::Report; - use std::{io, path::Path}; + use crate::diagnostics::{Report, SourceManager}; impl KernelLibrary { /// Write the library to a target file @@ -542,13 +551,17 @@ mod use_std_kernel { ) -> io::Result<()> { self.library.write_to_file(path, options) } - /// Read a directory and recursively create modules from its `masm` files. - /// - /// For every directory, concatenate the module path with the dir name and proceed. + + /// Create a [KernelLibrary] from a standard Miden Assembly project layout. /// - /// For every file, pick and compile the ones with `masm` extension; skip otherwise. - pub fn from_dir(path: impl AsRef) -> Result { - let library = CompiledLibrary::from_dir(path, LibraryNamespace::Kernel)?; + /// This is essentially a wrapper around [CompiledLibrary::from_dir], which then validates + /// that the resulting [CompiledLibrary] is a valid [KernelLibrary]. + pub fn from_dir( + path: impl AsRef, + source_manager: Arc, + ) -> Result { + let library = + CompiledLibrary::from_dir(path, LibraryNamespace::Kernel, source_manager)?; Ok(Self::try_from(library)?) } diff --git a/assembly/src/library/tests.rs b/assembly/src/library/tests.rs index 103ad0aadf..4a6680363c 100644 --- a/assembly/src/library/tests.rs +++ b/assembly/src/library/tests.rs @@ -1,27 +1,27 @@ -use alloc::{string::ToString, sync::Arc, vec::Vec}; +use alloc::{string::ToString, vec::Vec}; use vm_core::utils::SliceReader; -use super::LibraryPath; +use super::*; use crate::{ ast::{AstSerdeOptions, Module, ModuleKind, ProcedureName}, - diagnostics::{IntoDiagnostic, Report, SourceFile}, - library::CompiledLibrary, + diagnostics::{IntoDiagnostic, Report}, testing::TestContext, Assembler, Deserializable, }; macro_rules! parse_module { - ($path:literal, $source:expr) => {{ + ($context:expr, $path:literal, $source:expr) => {{ let path = LibraryPath::new($path).into_diagnostic()?; - let source_file = Arc::from(SourceFile::new(concat!("test", line!()), $source.to_string())); + let source_file = + $context.source_manager().load(concat!("test", line!()), $source.to_string()); Module::parse(path, ModuleKind::Library, source_file)? }}; } #[test] fn masl_locations_serialization() -> Result<(), Report> { - let _context = TestContext::new(); + let context = TestContext::new(); // declare foo module let foo = r#" export.foo @@ -31,7 +31,7 @@ fn masl_locations_serialization() -> Result<(), Report> { mul end "#; - let foo = parse_module!("test::foo", foo); + let foo = parse_module!(&context, "test::foo", foo); // declare bar module let bar = r#" @@ -42,11 +42,13 @@ fn masl_locations_serialization() -> Result<(), Report> { mul end "#; - let bar = parse_module!("test::bar", bar); + let bar = parse_module!(&context, "test::bar", bar); let modules = [foo, bar]; // serialize/deserialize the bundle with locations - let bundle = Assembler::default().assemble_library(modules.iter().cloned()).unwrap(); + let bundle = Assembler::new(context.source_manager()) + .assemble_library(modules.iter().cloned()) + .unwrap(); let mut bytes = Vec::new(); bundle.write_into_with_options(&mut bytes, AstSerdeOptions::new(true, true)); @@ -54,7 +56,9 @@ fn masl_locations_serialization() -> Result<(), Report> { assert_eq!(bundle, deserialized); // serialize/deserialize the bundle without locations - let bundle = Assembler::default().assemble_library(modules.iter().cloned()).unwrap(); + let bundle = Assembler::new(context.source_manager()) + .assemble_library(modules.iter().cloned()) + .unwrap(); // serialize/deserialize the bundle let mut bytes = Vec::new(); @@ -67,18 +71,20 @@ fn masl_locations_serialization() -> Result<(), Report> { #[test] fn get_module_by_path() -> Result<(), Report> { - let _context = TestContext::new(); + let context = TestContext::new(); // declare foo module let foo_source = r#" export.foo add end "#; - let foo = parse_module!("test::foo", foo_source); + let foo = parse_module!(&context, "test::foo", foo_source); let modules = [foo]; // create the bundle with locations - let bundle = Assembler::default().assemble_library(modules.iter().cloned()).unwrap(); + let bundle = Assembler::new(context.source_manager()) + .assemble_library(modules.iter().cloned()) + .unwrap(); let foo_module_info = bundle.module_infos().next().unwrap(); assert_eq!(foo_module_info.path(), &LibraryPath::new("test::foo").unwrap()); diff --git a/assembly/src/parser/error.rs b/assembly/src/parser/error.rs index ce2acdb3d6..d831148504 100644 --- a/assembly/src/parser/error.rs +++ b/assembly/src/parser/error.rs @@ -5,7 +5,7 @@ use alloc::{ use core::{fmt, ops::Range}; use super::{ParseError, SourceSpan}; -use crate::diagnostics::Diagnostic; +use crate::{diagnostics::Diagnostic, SourceId}; // LITERAL ERROR KIND // ================================================================================================ @@ -295,51 +295,50 @@ impl PartialEq for ParsingError { } } -impl From for ParsingError { - fn from(err: core::str::Utf8Error) -> Self { +impl ParsingError { + pub fn from_utf8_error(source_id: SourceId, err: core::str::Utf8Error) -> Self { let start = u32::try_from(err.valid_up_to()).ok().unwrap_or(u32::MAX); match err.error_len() { None => Self::IncompleteUtf8 { - span: SourceSpan::at(start), + span: SourceSpan::at(source_id, start), }, Some(len) => Self::InvalidUtf8 { - span: SourceSpan::new(start..(start + len as u32)), + span: SourceSpan::new(source_id, start..(start + len as u32)), }, } } -} -impl<'a> From> for ParsingError { - fn from(err: ParseError) -> Self { + pub fn from_parse_error(source_id: SourceId, err: ParseError<'_>) -> Self { use super::Token; + match err { ParseError::InvalidToken { location: at } => Self::InvalidToken { - span: SourceSpan::from(at..at), + span: SourceSpan::at(source_id, at), }, ParseError::UnrecognizedToken { token: (l, Token::Eof, r), expected, } => Self::UnrecognizedEof { - span: SourceSpan::from(l..r), + span: SourceSpan::new(source_id, l..r), expected: simplify_expected_tokens(expected), }, ParseError::UnrecognizedToken { token: (l, tok, r), expected, } => Self::UnrecognizedToken { - span: SourceSpan::from(l..r), + span: SourceSpan::new(source_id, l..r), token: tok.to_string(), expected: simplify_expected_tokens(expected), }, ParseError::ExtraToken { token: (l, tok, r) } => Self::ExtraToken { - span: SourceSpan::from(l..r), + span: SourceSpan::new(source_id, l..r), token: tok.to_string(), }, ParseError::UnrecognizedEof { location: at, expected, } => Self::UnrecognizedEof { - span: SourceSpan::from(at..at), + span: SourceSpan::new(source_id, at..at), expected: simplify_expected_tokens(expected), }, ParseError::User { error } => error, diff --git a/assembly/src/parser/grammar.lalrpop b/assembly/src/parser/grammar.lalrpop index cd5861d4ae..a8be8ce46d 100644 --- a/assembly/src/parser/grammar.lalrpop +++ b/assembly/src/parser/grammar.lalrpop @@ -245,11 +245,11 @@ Form: Form = { Doc: Form = { =>? { if doc.as_bytes().len() > u16::MAX as usize { - Err(ParseError::User { error: ParsingError::DocsTooLarge { span: span!(l, r) } }) + Err(ParseError::User { error: ParsingError::DocsTooLarge { span: span!(source_file.id(), l, r) } }) } else { match doc { - DocumentationType::Module(doc) => Ok(Form::ModuleDoc(Span::new(span!(l, r), doc))), - DocumentationType::Form(doc) => Ok(Form::Doc(Span::new(span!(l, r), doc))), + DocumentationType::Module(doc) => Ok(Form::ModuleDoc(Span::new(span!(source_file.id(), l, r), doc))), + DocumentationType::Form(doc) => Ok(Form::Doc(Span::new(span!(source_file.id(), l, r), doc))), } } } @@ -259,15 +259,15 @@ Import: Form = { "use" "." " )?> =>? { match import_path { (module_name, None) => Err(ParseError::User { - error: ParsingError::UnqualifiedImport { span: span!(l, r) }, + error: ParsingError::UnqualifiedImport { span: span!(source_file.id(), l, r) }, }), (module_name, Some(path)) => { let path = path.append_ident(module_name.clone()) .map_err(|error| { - ParseError::User { error: ParsingError::InvalidLibraryPath { span: span!(l, r), message: error.to_string() }} + ParseError::User { error: ParsingError::InvalidLibraryPath { span: span!(source_file.id(), l, r), message: error.to_string() }} })?; let name = alias.unwrap_or(module_name); - Ok(Form::Import(Import { span: span!(l, r), name, path, uses: 0 })) + Ok(Form::Import(Import { span: span!(source_file.id(), l, r), name, path, uses: 0 })) } } } @@ -276,7 +276,7 @@ Import: Form = { Const: Form = { "const" "." "=" => { Form::Constant(Constant::new( - span!(l, r), + span!(source_file.id(), l, r), name, value, )) @@ -285,7 +285,7 @@ Const: Form = { Begin: Form = { "begin" "end" => { - Form::Begin(Block::new(span!(l, r), body)) + Form::Begin(Block::new(span!(source_file.id(), l, r), body)) } } @@ -293,17 +293,17 @@ Proc: Form = { "." > "end" =>? { let num_locals = num_locals.unwrap_or(0); let procedure = Procedure::new( - span!(l, r), + span!(source_file.id(), l, r), visibility, name, num_locals, body - ).with_source_file(Some(source_file.clone())); + ); Ok(Form::Procedure(Export::Procedure(procedure))) }, "export" "." " )?> =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); let alias = match name { InvocationTarget::ProcedureName(_) => return Err(ParseError::User { error: ParsingError::UnqualifiedImport { span }, @@ -341,7 +341,6 @@ Proc: Form = { ProcedureAlias::new(export_name, AliasTarget::AbsoluteProcedurePath(target)) } }; - let alias = alias.with_source_file(Some(source_file.clone())); Ok(Form::Procedure(Export::Alias(alias))) } } @@ -356,7 +355,7 @@ Visibility: Visibility = { // ================================================================================================ Block: Block = { - => Block::new(span!(l, r), body), + => Block::new(span!(source_file.id(), l, r), body), } #[inline] @@ -364,7 +363,7 @@ Ops: Vec = { =>? { let ops = ops.into_iter().flat_map(|ops| ops.into_iter()).collect::>(); if ops.len() > u16::MAX as usize { - Err(ParseError::User { error: ParsingError::CodeBlockTooBig { span: span!(l, r) } }) + Err(ParseError::User { error: ParsingError::CodeBlockTooBig { span: span!(source_file.id(), l, r) } }) } else { Ok(ops) } @@ -381,7 +380,7 @@ Op: SmallOpsVec = { IfElse: Op = { // Handles the edge case of a code generator emitting an empty "then" block "if" "." "else" "end" => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); let then_blk = Block::new(span, vec![Op::Inst(Span::new(span, Instruction::Nop))]); // If false-conditioned, swap the blocks if cond { @@ -392,7 +391,7 @@ IfElse: Op = { }, "if" "." "else" "end" => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); let else_blk = else_blk.unwrap_or_else(|| Block::new(span, vec![Op::Inst(Span::new(span, Instruction::Nop))])); // If false-conditioned, swap the blocks if cond { @@ -403,7 +402,7 @@ IfElse: Op = { }, "if" "." "end" => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); let else_blk = Block::new(span, vec![Op::Inst(Span::new(span, Instruction::Nop))]); // If false-conditioned, swap the blocks if cond { @@ -422,13 +421,13 @@ Condition: bool = { While: Op = { "while" "." "true" "end" => { - Op::While { span: span!(l, r), body } + Op::While { span: span!(source_file.id(), l, r), body } }, } Repeat: Op = { "repeat" "." "end" =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); u32::try_from(count) .map_err(|error| ParseError::User { error: ParsingError::ImmediateOutOfRange { span, range: 1..(u32::MAX as usize) } }) .and_then(|count| { @@ -441,7 +440,7 @@ Repeat: Op = { #[inline] Instruction: SmallOpsVec = { - => smallvec![Op::Inst(Span::new(span!(l, r), inst))], + => smallvec![Op::Inst(Span::new(span!(source_file.id(), l, r), inst))], // For instructions which may fold to zero or multiple instructions; // or for instruction macros, which expand to multiple instructions, // this is the rule under which those instructions should be handled @@ -610,21 +609,21 @@ ProcRef: Instruction = { #[inline] FoldableInstWithFeltImmediate: SmallOpsVec = { "eq" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::EqImm(imm)))], None => smallvec![Op::Inst(Span::new(span, Instruction::Eq))], } }, "neq" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::NeqImm(imm)))], None => smallvec![Op::Inst(Span::new(span, Instruction::Neq))], } }, "lt" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -636,7 +635,7 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "lte" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -648,7 +647,7 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "gt" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -660,7 +659,7 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "gte" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -672,7 +671,7 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "add" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == Felt::ZERO => smallvec![], Some(imm) if imm == Felt::ONE => smallvec![Op::Inst(Span::new(span, Instruction::Incr))], @@ -681,7 +680,7 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "sub" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == Felt::ZERO => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::SubImm(imm)))], @@ -689,7 +688,7 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "mul" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == Felt::ZERO => smallvec![Op::Inst(Span::new(span, Instruction::Drop)), Op::Inst(Span::new(span, Instruction::PushU8(0)))], Some(imm) if imm == Felt::ONE => smallvec![], @@ -698,9 +697,9 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { } }, "div" > =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { - Some(imm) if imm == Felt::ZERO => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(l, r) } }), + Some(imm) if imm == Felt::ZERO => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(source_file.id(), l, r) } }), Some(imm) if imm == Felt::ONE => Ok(smallvec![]), Some(imm) => Ok(smallvec![Op::Inst(Span::new(span, Instruction::DivImm(imm)))]), None => Ok(smallvec![Op::Inst(Span::new(span, Instruction::Div))]), @@ -711,32 +710,32 @@ FoldableInstWithFeltImmediate: SmallOpsVec = { #[inline] FoldableInstWithU32Immediate: SmallOpsVec = { "u32div" > =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { - Some(imm) if imm == 0 => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(l, r) } }), + Some(imm) if imm == 0 => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(source_file.id(), l, r) } }), Some(imm) if imm == 1 => Ok(smallvec![]), Some(imm) => Ok(smallvec![Op::Inst(Span::new(span, Instruction::U32DivImm(imm)))]), None => Ok(smallvec![Op::Inst(Span::new(span, Instruction::U32Div))]), } }, "u32divmod" > =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { - Some(imm) if imm == 0 => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(l, r) } }), + Some(imm) if imm == 0 => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(source_file.id(), l, r) } }), Some(imm) => Ok(smallvec![Op::Inst(Span::new(span, Instruction::U32DivModImm(imm)))]), None => Ok(smallvec![Op::Inst(Span::new(span, Instruction::U32DivMod))]), } }, "u32mod" > =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { - Some(imm) if imm == 0 => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(l, r) } }), + Some(imm) if imm == 0 => Err(ParseError::User { error: ParsingError::DivisionByZero { span: span!(source_file.id(), l, r) } }), Some(imm) => Ok(smallvec![Op::Inst(Span::new(span, Instruction::U32ModImm(imm)))]), None => Ok(smallvec![Op::Inst(Span::new(span, Instruction::U32Mod))]), } }, "u32and" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![Op::Inst(Span::new(span, Instruction::Drop)), Op::Inst(Span::new(span, Instruction::PushU8(0)))], Some(imm) => { @@ -749,7 +748,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32or" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => { @@ -762,7 +761,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32xor" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => { @@ -775,7 +774,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32not" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -787,7 +786,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32wrapping_add" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32WrappingAddImm(imm)))], @@ -795,7 +794,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32wrapping_sub" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32WrappingSubImm(imm)))], @@ -803,7 +802,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32wrapping_mul" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![Op::Inst(Span::new(span, Instruction::Drop)), Op::Inst(Span::new(span, Instruction::PushU8(0)))], Some(imm) if imm == 1 => smallvec![], @@ -812,28 +811,28 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32overflowing_add" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32OverflowingAddImm(imm)))], None => smallvec![Op::Inst(Span::new(span, Instruction::U32OverflowingAdd))], } }, "u32overflowing_sub" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32OverflowingSubImm(imm)))], None => smallvec![Op::Inst(Span::new(span, Instruction::U32OverflowingSub))], } }, "u32overflowing_mul" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32OverflowingMulImm(imm)))], None => smallvec![Op::Inst(Span::new(span, Instruction::U32OverflowingMul))], } }, "u32shl" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32ShlImm(imm)))], @@ -841,7 +840,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32shr" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32ShrImm(imm)))], @@ -849,7 +848,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32rotl" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32RotlImm(imm)))], @@ -857,7 +856,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32rotr" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) if imm == 0 => smallvec![], Some(imm) => smallvec![Op::Inst(Span::new(span, Instruction::U32RotrImm(imm)))], @@ -865,7 +864,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32lt" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -877,7 +876,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32lte" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -889,7 +888,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32gt" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -901,7 +900,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32gte" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -913,7 +912,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32min" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -925,7 +924,7 @@ FoldableInstWithU32Immediate: SmallOpsVec = { } }, "u32max" > => { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); match imm { Some(imm) => { match imm { @@ -962,7 +961,7 @@ InstWithLocalIndex: Instruction = { #[inline] InstWithStackIndex: Instruction = { - "adv_push" "." => Instruction::AdvPush(Immediate::Value(Span::new(span!(l, r), i))), + "adv_push" "." => Instruction::AdvPush(Immediate::Value(Span::new(span!(source_file.id(), l, r), i))), "dup" )?> =>? { let (span, idx) = i.map(|s| s.into_parts()).unwrap_or((SourceSpan::default(), 0)); Ok(match idx { @@ -1084,8 +1083,14 @@ InstWithBitSizeImmediate: Instruction = { Push: SmallOpsVec = { "push" "." > =>? { - let ops = values.into_iter().map(|imm| { + let ops = values.into_iter().enumerate().map(|(i, imm)| { let span = imm.span(); + // Include the "push" token in the first item's span + let span = if i == 0 { + span!(source_file.id(), l, span.end().to_u32()) + } else { + span + }; Op::Inst(Span::new(span, match imm { Immediate::Constant(name) => Instruction::Push(Immediate::Constant(name)), Immediate::Value(value) => { @@ -1100,7 +1105,7 @@ Push: SmallOpsVec = { })) }).collect::(); if ops.len() > 16 { - Err(ParseError::User { error: ParsingError::PushOverflow { span: span!(l, r), count: ops.len() } }) + Err(ParseError::User { error: ParsingError::PushOverflow { span: span!(source_file.id(), l, r), count: ops.len() } }) } else { Ok(ops) } @@ -1118,7 +1123,7 @@ Imm: Immediate = { #[inline] ImmValue: Immediate = { - => Immediate::Value(Span::new(span!(l, r), t)), + => Immediate::Value(Span::new(span!(source_file.id(), l, r), t)), => Immediate::Constant(<>), } @@ -1155,7 +1160,7 @@ RawU8: u8 = { U8: Span = { =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); u8::try_from(n).map_err(|error| ParseError::User { error: ParsingError::ImmediateOutOfRange { span, range: 0..(u8::MAX as usize + 1) }, }).map(|n| Span::new(span, n)) @@ -1165,7 +1170,7 @@ U8: Span = { U16: u16 = { =>? { u16::try_from(n).map_err(|error| ParseError::User { - error: ParsingError::ImmediateOutOfRange { span: span!(l, r), range: 0..(u16::MAX as usize + 1) }, + error: ParsingError::ImmediateOutOfRange { span: span!(source_file.id(), l, r), range: 0..(u16::MAX as usize + 1) }, }) } } @@ -1173,7 +1178,7 @@ U16: u16 = { U32: u32 = { =>? { u32::try_from(n).map_err(|error| ParseError::User { - error: ParsingError::InvalidLiteral { span: span!(l, r), kind: LiteralErrorKind::U32Overflow }, + error: ParsingError::InvalidLiteral { span: span!(source_file.id(), l, r), kind: LiteralErrorKind::U32Overflow }, }) }, @@ -1182,7 +1187,7 @@ U32: u32 = { HexEncodedValue::U8(v) => Ok(v as u32), HexEncodedValue::U16(v) => Ok(v as u32), HexEncodedValue::U32(v) => Ok(v), - _ => Err(ParseError::User { error: ParsingError::InvalidLiteral { span: span!(l, r), kind: LiteralErrorKind::U32Overflow } }), + _ => Err(ParseError::User { error: ParsingError::InvalidLiteral { span: span!(source_file.id(), l, r), kind: LiteralErrorKind::U32Overflow } }), } }, @@ -1198,9 +1203,9 @@ U32: u32 = { MastRoot: Span = { =>? { match value { - HexEncodedValue::Word(word) => Ok(Span::new(span!(l, r), RpoDigest::from(word))), + HexEncodedValue::Word(word) => Ok(Span::new(span!(source_file.id(), l, r), RpoDigest::from(word))), _ => { - Err(ParseError::User { error: ParsingError::InvalidMastRoot { span: span!(l, r) } }) + Err(ParseError::User { error: ParsingError::InvalidMastRoot { span: span!(source_file.id(), l, r) } }) } } } @@ -1257,14 +1262,14 @@ BitSize: u8 = { } IntOrHexImm: Immediate = { - => Immediate::Value(Span::new(span!(l, r), value)), + => Immediate::Value(Span::new(span!(source_file.id(), l, r), value)), => Immediate::Constant(<>), } IntOrHex: HexEncodedValue = { =>? { if n > Felt::MODULUS { - return Err(ParseError::User { error: ParsingError::InvalidLiteral { span: span!(l, r), kind: LiteralErrorKind::FeltOverflow } }); + return Err(ParseError::User { error: ParsingError::InvalidLiteral { span: span!(source_file.id(), l, r), kind: LiteralErrorKind::FeltOverflow } }); } if n <= (u8::MAX as u64) { Ok(HexEncodedValue::U8(n as u8)) @@ -1283,7 +1288,7 @@ IntOrHex: HexEncodedValue = { Felt: Felt = { =>? { if n > Felt::MODULUS { - return Err(ParseError::User { error: ParsingError::InvalidLiteral { span: span!(l, r), kind: LiteralErrorKind::FeltOverflow } }); + return Err(ParseError::User { error: ParsingError::InvalidLiteral { span: span!(source_file.id(), l, r), kind: LiteralErrorKind::FeltOverflow } }); } Ok(Felt::new(n)) }, @@ -1295,7 +1300,7 @@ Felt: Felt = { HexEncodedValue::U32(v) => Felt::new(v as u64), HexEncodedValue::Felt(v) => v, HexEncodedValue::Word(_) => return Err(ParseError::User { - error: ParsingError::InvalidHexLiteral { span: span!(l, r), kind: HexErrorKind::Overflow }, + error: ParsingError::InvalidHexLiteral { span: span!(source_file.id(), l, r), kind: HexErrorKind::Overflow }, }), }) } @@ -1323,7 +1328,7 @@ QuotedProcedureName: ProcedureName = { interned.insert(name.clone()); name }); - let id = Ident::new_unchecked(Span::new(span!(l, r), name)); + let id = Ident::new_unchecked(Span::new(span!(source_file.id(), l, r), name)); ProcedureName::new_unchecked(id) } } @@ -1384,7 +1389,7 @@ BareIdent: Ident = { interned.insert(name.clone()); name }); - Ident::new_unchecked(Span::new(span!(l, r), name)) + Ident::new_unchecked(Span::new(span!(source_file.id(), l, r), name)) }, OpcodeName, @@ -1392,7 +1397,7 @@ BareIdent: Ident = { MaybeQualifiedName: (Ident, Option) = { > =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); let name = components.pop().unwrap(); if components.is_empty() { Ok((name, None)) @@ -1425,7 +1430,7 @@ MaybeQualifiedName: (Ident, Option) = { QualifiedName: (Ident, LibraryPath) = { "::")> > =>? { - let span = span!(l, r); + let span = span!(source_file.id(), l, r); let name = components.pop().unwrap(); let ns = match first.as_str() { @@ -1462,7 +1467,7 @@ OpcodeName: Ident = { interned.insert(name.clone()); name }); - Ident::new_unchecked(Span::new(span!(l, r), name)) + Ident::new_unchecked(Span::new(span!(source_file.id(), l, r), name)) } } @@ -1607,12 +1612,12 @@ Opcode: &'static str = { ConstantExpr: ConstantExpr = { "+" =>? { - let expr = ConstantExpr::BinaryOp { span: span!(l, r), op: ConstantOp::Add, lhs: Box::new(x), rhs: Box::new(y) }; + let expr = ConstantExpr::BinaryOp { span: span!(source_file.id(), l, r), op: ConstantOp::Add, lhs: Box::new(x), rhs: Box::new(y) }; expr.try_fold().map_err(|error| ParseError::User { error }) }, "-" =>? { - let expr = ConstantExpr::BinaryOp { span: span!(l, r), op: ConstantOp::Sub, lhs: Box::new(x), rhs: Box::new(y) }; + let expr = ConstantExpr::BinaryOp { span: span!(source_file.id(), l, r), op: ConstantOp::Sub, lhs: Box::new(x), rhs: Box::new(y) }; expr.try_fold().map_err(|error| ParseError::User { error }) }, @@ -1621,17 +1626,17 @@ ConstantExpr: ConstantExpr = { ConstantExpr100: ConstantExpr = { "*" =>? { - let expr = ConstantExpr::BinaryOp { span: span!(l, r), op: ConstantOp::Mul, lhs: Box::new(x), rhs: Box::new(y) }; + let expr = ConstantExpr::BinaryOp { span: span!(source_file.id(), l, r), op: ConstantOp::Mul, lhs: Box::new(x), rhs: Box::new(y) }; expr.try_fold().map_err(|error| ParseError::User { error }) }, "/" =>? { - let expr = ConstantExpr::BinaryOp { span: span!(l, r), op: ConstantOp::Div, lhs: Box::new(x), rhs: Box::new(y) }; + let expr = ConstantExpr::BinaryOp { span: span!(source_file.id(), l, r), op: ConstantOp::Div, lhs: Box::new(x), rhs: Box::new(y) }; expr.try_fold().map_err(|error| ParseError::User { error }) }, "//" =>? { - let expr = ConstantExpr::BinaryOp { span: span!(l, r), op: ConstantOp::IntDiv, lhs: Box::new(x), rhs: Box::new(y) }; + let expr = ConstantExpr::BinaryOp { span: span!(source_file.id(), l, r), op: ConstantOp::IntDiv, lhs: Box::new(x), rhs: Box::new(y) }; expr.try_fold().map_err(|error| ParseError::User { error }) }, @@ -1646,7 +1651,7 @@ ConstantName: Ident = { interned.insert(name.clone()); name }); - Ident::new_unchecked(Span::new(span!(l, r), name)) + Ident::new_unchecked(Span::new(span!(source_file.id(), l, r), name)) } } diff --git a/assembly/src/parser/lexer.rs b/assembly/src/parser/lexer.rs index 8abbd34e95..9efa86ba99 100644 --- a/assembly/src/parser/lexer.rs +++ b/assembly/src/parser/lexer.rs @@ -1,10 +1,13 @@ -use crate::Felt; use alloc::string::String; use core::{num::IntErrorKind, ops::Range}; use super::{ BinEncodedValue, BinErrorKind, DocumentationType, HexEncodedValue, HexErrorKind, - LiteralErrorKind, ParsingError, Scanner, SourceSpan, Token, + LiteralErrorKind, ParsingError, Scanner, Token, +}; +use crate::{ + diagnostics::{ByteOffset, SourceId, SourceSpan}, + Felt, }; /// The value produced by the [Lexer] when iterated @@ -55,6 +58,9 @@ macro_rules! pop2 { /// guarantee that parsing them will produce meaningful results, it is primarily to assist in /// gathering as many errors as possible. pub struct Lexer<'input> { + /// The [SourceId] of the file being lexed, for use in producing spans in lexer diagnostics + source_id: SourceId, + /// The scanner produces a sequence of chars + location, and can be controlled /// The location type is usize scanner: Scanner<'input>, @@ -90,10 +96,11 @@ pub struct Lexer<'input> { impl<'input> Lexer<'input> { /// Produces an instance of the lexer with the lexical analysis to be performed on the `input` /// string. Note that no lexical analysis occurs until the lexer has been iterated over. - pub fn new(scanner: Scanner<'input>) -> Self { + pub fn new(source_id: SourceId, scanner: Scanner<'input>) -> Self { let start = scanner.start(); let keywords = Token::keyword_searcher(); let mut lexer = Self { + source_id, scanner, token: Token::Eof, token_start: start, @@ -206,7 +213,7 @@ impl<'input> Lexer<'input> { fn span(&self) -> SourceSpan { assert!(self.token_start <= self.token_end, "invalid range"); assert!(self.token_end <= u32::MAX as usize, "file too large"); - SourceSpan::from((self.token_start as u32)..(self.token_end as u32)) + SourceSpan::new(self.source_id, (self.token_start as u32)..(self.token_end as u32)) } #[inline] @@ -411,18 +418,18 @@ impl<'input> Lexer<'input> { // Skip quotation mark self.skip(); - let quote_size = '"'.len_utf8() as u32; + let quote_size = ByteOffset::from_char_len('"'); loop { match self.read() { '\0' | '\n' => { break Err(ParsingError::UnclosedQuote { - start: SourceSpan::at(self.span().start() as u32), + start: SourceSpan::at(self.source_id, self.span().start()), }); } '"' => { let span = self.span(); - let start = span.start() as u32 + quote_size; - let span = SourceSpan::from(start..(span.end() as u32)); + let start = span.start() + quote_size; + let span = SourceSpan::new(self.source_id, start..span.end()); self.skip(); break Ok(Token::QuotedIdent(self.slice_span(span))); @@ -436,9 +443,9 @@ impl<'input> Lexer<'input> { continue; } c => { - let loc = self.span().end() - c.len_utf8(); + let loc = self.span().end() - ByteOffset::from_char_len(c); break Err(ParsingError::InvalidIdentCharacter { - span: SourceSpan::at(loc as u32), + span: SourceSpan::at(self.source_id, loc), }); } } @@ -522,11 +529,11 @@ impl<'input> Lexer<'input> { } let span = self.span(); - let start = span.start() as u32; - let digit_start = start + 2; - let end = span.end() as u32; - let span = SourceSpan::from(start..end); - let value = parse_hex(span, self.slice_span(digit_start..end))?; + let start = span.start(); + let end = span.end(); + let digit_start = start.to_u32() + 2; + let span = SourceSpan::new(span.source_id(), start..end); + let value = parse_hex(span, self.slice_span(digit_start..end.to_u32()))?; Ok(Token::HexValue(value)) } @@ -544,11 +551,11 @@ impl<'input> Lexer<'input> { } let span = self.span(); - let start = span.start() as u32; - let digit_start = start + 2; - let end = span.end() as u32; - let span = SourceSpan::from(start..end); - let value = parse_bin(span, self.slice_span(digit_start..end))?; + let start = span.start(); + let digit_start = start.to_u32() + 2; + let end = span.end(); + let span = SourceSpan::new(span.source_id(), start..end); + let value = parse_bin(span, self.slice_span(digit_start..end.to_u32()))?; Ok(Token::BinValue(value)) } } diff --git a/assembly/src/parser/location.rs b/assembly/src/parser/location.rs deleted file mode 100644 index 9ff1c1fd5f..0000000000 --- a/assembly/src/parser/location.rs +++ /dev/null @@ -1,57 +0,0 @@ -use crate::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable}; -use core::fmt; - -// SOURCE LOCATION -// ================================================================================================ - -/// A struct containing information about the location of a source item. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct SourceLocation { - // TODO add uri - line: u32, - column: u32, -} - -impl SourceLocation { - /// Creates a new instance of [SourceLocation]. - pub const fn new(line: u32, column: u32) -> Self { - Self { line, column } - } - - /// Returns the line of the location. - pub const fn line(&self) -> u32 { - self.line - } - - /// Moves the column by the given offset. - pub fn move_column(&mut self, offset: u32) { - self.column += offset; - } -} - -impl Default for SourceLocation { - fn default() -> Self { - Self { line: 1, column: 1 } - } -} - -impl fmt::Display for SourceLocation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "[{}:{}]", self.line, self.column) - } -} - -impl Serializable for SourceLocation { - fn write_into(&self, target: &mut W) { - target.write_u32(self.line); - target.write_u32(self.column); - } -} - -impl Deserializable for SourceLocation { - fn read_from(source: &mut R) -> Result { - let line = source.read_u32()?; - let column = source.read_u32()?; - Ok(Self { line, column }) - } -} diff --git a/assembly/src/parser/mod.rs b/assembly/src/parser/mod.rs index 08618e2578..34030910fe 100644 --- a/assembly/src/parser/mod.rs +++ b/assembly/src/parser/mod.rs @@ -1,10 +1,10 @@ /// Simple macro used in the grammar definition for constructing spans macro_rules! span { - ($l:expr, $r:expr) => { - crate::SourceSpan::new($l..$r) + ($id:expr, $l:expr, $r:expr) => { + crate::SourceSpan::new($id, $l..$r) }; - ($i:expr) => { - crate::SourceSpan::new($i..$i) + ($id:expr, $i:expr) => { + crate::SourceSpan::at($id, $i) }; } @@ -16,22 +16,18 @@ lalrpop_util::lalrpop_mod!( mod error; mod lexer; -mod location; mod scanner; -mod span; mod token; pub use self::error::{BinErrorKind, HexErrorKind, LiteralErrorKind, ParsingError}; pub use self::lexer::Lexer; -pub use self::location::SourceLocation; pub use self::scanner::Scanner; -pub use self::span::{SourceSpan, Span, Spanned}; pub use self::token::{BinEncodedValue, DocumentationType, HexEncodedValue, Token}; use crate::{ ast, - diagnostics::{Report, SourceFile}, - sema, LibraryPath, + diagnostics::{Report, SourceFile, SourceSpan, Span, Spanned}, + sema, LibraryPath, SourceManager, }; use alloc::{boxed::Box, collections::BTreeSet, string::ToString, sync::Arc, vec::Vec}; @@ -42,9 +38,6 @@ type ParseError<'a> = lalrpop_util::ParseError, ParsingError>; /// This is a wrapper around the lower-level parser infrastructure which handles orchestrating all /// of the pieces needed to parse a [ast::Module] from source, and run semantic analysis on it. -/// -/// In the vast majority of cases though, you will want to use the more ergonomic -/// [ast::Module::parse_str] or [ast::Module::parse_file] APIs instead. #[derive(Default)] pub struct ModuleParser { /// The kind of module we're parsing. @@ -101,18 +94,23 @@ impl ModuleParser { /// Parse a [ast::Module], `name`, from `path`. #[cfg(feature = "std")] - pub fn parse_file

(&mut self, name: LibraryPath, path: P) -> Result, Report> + pub fn parse_file

( + &mut self, + name: LibraryPath, + path: P, + source_manager: &dyn SourceManager, + ) -> Result, Report> where P: AsRef, { use crate::diagnostics::{IntoDiagnostic, WrapErr}; + use vm_core::debuginfo::SourceManagerExt; let path = path.as_ref(); - let filename = path.to_string_lossy(); - let source = std::fs::read_to_string(path) + let source_file = source_manager + .load_file(path) .into_diagnostic() - .wrap_err_with(|| format!("failed to parse module from '{filename}'"))?; - let source_file = Arc::new(SourceFile::new(filename, source)); + .wrap_err_with(|| format!("failed to load source file from '{}'", path.display()))?; self.parse(name, source_file) } @@ -121,9 +119,13 @@ impl ModuleParser { &mut self, name: LibraryPath, source: impl ToString, + source_manager: &dyn SourceManager, ) -> Result, Report> { - let source = source.to_string(); - let source_file = Arc::new(SourceFile::new(name.path(), source)); + use vm_core::debuginfo::SourceContent; + + let path = Arc::from(name.path().into_owned().into_boxed_str()); + let content = SourceContent::new(Arc::clone(&path), source.to_string().into_boxed_str()); + let source_file = source_manager.load_from_raw_parts(path, content); self.parse(name, source_file) } } @@ -146,11 +148,12 @@ fn parse_forms_internal( source: Arc, interned: &mut BTreeSet>, ) -> Result, ParsingError> { - let scanner = Scanner::new(source.inner().as_ref()); - let lexer = Lexer::new(scanner); + let source_id = source.id(); + let scanner = Scanner::new(source.as_str()); + let lexer = Lexer::new(source_id, scanner); grammar::FormsParser::new() .parse(&source, interned, core::marker::PhantomData, lexer) - .map_err(ParsingError::from) + .map_err(|err| ParsingError::from_parse_error(source_id, err)) } // TESTS @@ -159,13 +162,15 @@ fn parse_forms_internal( #[cfg(test)] mod tests { use super::*; + use crate::SourceId; use vm_core::assert_matches; // This test checks the lexer behavior with regard to tokenizing `exp(.u?[\d]+)?` #[test] fn lex_exp() { + let source_id = SourceId::default(); let scanner = Scanner::new("begin exp.u9 end"); - let mut lexer = Lexer::new(scanner).map(|result| result.map(|(_, t, _)| t)); + let mut lexer = Lexer::new(source_id, scanner).map(|result| result.map(|(_, t, _)| t)); assert_matches!(lexer.next(), Some(Ok(Token::Begin))); assert_matches!(lexer.next(), Some(Ok(Token::ExpU))); assert_matches!(lexer.next(), Some(Ok(Token::Int(n))) if n == 9); @@ -174,6 +179,7 @@ mod tests { #[test] fn lex_block() { + let source_id = SourceId::default(); let scanner = Scanner::new( "\ const.ERR1=1 @@ -185,7 +191,7 @@ begin end ", ); - let mut lexer = Lexer::new(scanner).map(|result| result.map(|(_, t, _)| t)); + let mut lexer = Lexer::new(source_id, scanner).map(|result| result.map(|(_, t, _)| t)); assert_matches!(lexer.next(), Some(Ok(Token::Const))); assert_matches!(lexer.next(), Some(Ok(Token::Dot))); assert_matches!(lexer.next(), Some(Ok(Token::ConstantIdent("ERR1")))); @@ -209,6 +215,7 @@ end #[test] fn lex_emit() { + let source_id = SourceId::default(); let scanner = Scanner::new( "\ begin @@ -217,7 +224,7 @@ begin end ", ); - let mut lexer = Lexer::new(scanner).map(|result| result.map(|(_, t, _)| t)); + let mut lexer = Lexer::new(source_id, scanner).map(|result| result.map(|(_, t, _)| t)); assert_matches!(lexer.next(), Some(Ok(Token::Begin))); assert_matches!(lexer.next(), Some(Ok(Token::Push))); assert_matches!(lexer.next(), Some(Ok(Token::Dot))); diff --git a/assembly/src/sema/context.rs b/assembly/src/sema/context.rs index b3c8e0294d..685936a7c7 100644 --- a/assembly/src/sema/context.rs +++ b/assembly/src/sema/context.rs @@ -1,7 +1,7 @@ use crate::{ ast::*, - diagnostics::{Diagnostic, Severity, SourceFile}, - Felt, Span, Spanned, + diagnostics::{Diagnostic, Severity}, + Felt, SourceFile, Span, Spanned, }; use alloc::{ collections::{BTreeMap, BTreeSet}, @@ -13,21 +13,21 @@ use super::{SemanticAnalysisError, SyntaxError}; /// This maintains the state for semantic analysis of a single [Module]. pub struct AnalysisContext { - source_code: Arc, /// A map of constants to the value of that constant constants: BTreeMap, procedures: BTreeSet, errors: Vec, + source_file: Arc, warnings_as_errors: bool, } impl AnalysisContext { - pub fn new(source_code: Arc) -> Self { + pub fn new(source_file: Arc) -> Self { Self { - source_code, constants: Default::default(), procedures: Default::default(), errors: Default::default(), + source_file, warnings_as_errors: false, } } @@ -41,10 +41,6 @@ impl AnalysisContext { self.warnings_as_errors } - pub fn source_file(&self) -> Arc { - self.source_code.clone() - } - pub fn register_procedure_name(&mut self, name: ProcedureName) { self.procedures.insert(name); } @@ -73,7 +69,7 @@ impl AnalysisContext { self.errors.push(err); let errors = core::mem::take(&mut self.errors); Err(SyntaxError { - input: self.source_code.clone(), + source_file: self.source_file.clone(), errors, }) } @@ -131,7 +127,7 @@ impl AnalysisContext { pub fn has_failed(&mut self) -> Result<(), SyntaxError> { if self.has_errors() { Err(SyntaxError { - input: self.source_file(), + source_file: self.source_file.clone(), errors: core::mem::take(&mut self.errors), }) } else { @@ -142,7 +138,7 @@ impl AnalysisContext { pub fn into_result(self) -> Result<(), SyntaxError> { if self.has_errors() { Err(SyntaxError { - input: self.source_code, + source_file: self.source_file.clone(), errors: self.errors, }) } else { @@ -158,7 +154,7 @@ impl AnalysisContext { if !self.errors.is_empty() { // Emit warnings to stderr let warning = Report::from(super::errors::SyntaxWarning { - input: self.source_code, + source_file: self.source_file, errors: self.errors, }); std::eprintln!("{}", warning); diff --git a/assembly/src/sema/errors.rs b/assembly/src/sema/errors.rs index 59b1852f4d..eb2e63d642 100644 --- a/assembly/src/sema/errors.rs +++ b/assembly/src/sema/errors.rs @@ -1,7 +1,4 @@ -use crate::{ - diagnostics::{Diagnostic, SourceFile}, - SourceSpan, -}; +use crate::{diagnostics::Diagnostic, SourceFile, SourceSpan}; use alloc::{sync::Arc, vec::Vec}; use core::fmt; @@ -20,7 +17,7 @@ use core::fmt; #[diagnostic(help("see emitted diagnostics for details"))] pub struct SyntaxError { #[source_code] - pub input: Arc, + pub source_file: Arc, #[related] pub errors: Vec, } @@ -35,7 +32,7 @@ pub struct SyntaxError { #[diagnostic(help("see below for details"))] pub struct SyntaxWarning { #[source_code] - pub input: Arc, + pub source_file: Arc, #[related] pub errors: Vec, } diff --git a/assembly/src/sema/mod.rs b/assembly/src/sema/mod.rs index 35c1d3d613..09ebf9ea68 100644 --- a/assembly/src/sema/mod.rs +++ b/assembly/src/sema/mod.rs @@ -31,7 +31,7 @@ pub fn analyze( let mut analyzer = AnalysisContext::new(source.clone()); analyzer.set_warnings_as_errors(warnings_as_errors); - let mut module = Box::new(Module::new(kind, path).with_source_file(Some(source))); + let mut module = Box::new(Module::new(kind, path).with_span(source.source_span())); let mut forms = VecDeque::from(forms); let mut docs = None; @@ -89,11 +89,9 @@ pub fn analyze( }, Form::Begin(body) if matches!(kind, ModuleKind::Executable) => { let docs = docs.take(); - let source_file = analyzer.source_file(); let procedure = Procedure::new(body.span(), Visibility::Public, ProcedureName::main(), 0, body) - .with_docs(docs) - .with_source_file(Some(source_file)); + .with_docs(docs); define_procedure(Export::Procedure(procedure), &mut module, &mut analyzer)?; } Form::Begin(body) => { diff --git a/assembly/src/testing.rs b/assembly/src/testing.rs index 95faeb56c7..a3817b3bfb 100644 --- a/assembly/src/testing.rs +++ b/assembly/src/testing.rs @@ -3,7 +3,7 @@ use crate::{ ast::{Form, Module, ModuleKind}, diagnostics::{ reporting::{set_hook, ReportHandlerOpts}, - Report, SourceFile, + Report, SourceFile, SourceManager, }, library::CompiledLibrary, Compile, CompileOptions, LibraryPath, RpoDigest, @@ -125,17 +125,11 @@ macro_rules! regex { /// the source file was constructed. #[macro_export] macro_rules! source_file { - ($source:literal) => { - ::alloc::sync::Arc::new($crate::diagnostics::SourceFile::new( - concat!("test", line!()), - $source.to_string(), - )) + ($context:expr, $source:literal) => { + $context.source_manager().load(concat!("test", line!()), $source.to_string()) }; - ($source:expr) => { - ::alloc::sync::Arc::new($crate::diagnostics::SourceFile::new( - concat!("test", line!()), - $source, - )) + ($context:expr, $source:expr) => { + $context.source_manager().load(concat!("test", line!()), $source.to_string()) }; } @@ -178,6 +172,7 @@ macro_rules! assert_diagnostic_lines { /// /// Some of the assertion macros defined above require a [TestContext], so be aware of that. pub struct TestContext { + source_manager: Arc, assembler: Assembler, } @@ -202,17 +197,32 @@ impl TestContext { { let _ = set_hook(Box::new(|_| Box::new(ReportHandlerOpts::new().build()))); } + let source_manager = Arc::new(crate::DefaultSourceManager::default()); + let assembler = Assembler::new(source_manager.clone()) + .with_debug_mode(true) + .with_warnings_as_errors(true); Self { - assembler: Assembler::default().with_debug_mode(true).with_warnings_as_errors(true), + source_manager, + assembler, } } + pub fn with_debug_info(mut self, yes: bool) -> Self { + self.assembler.set_debug_mode(yes); + self + } + + #[inline(always)] + pub fn source_manager(&self) -> Arc { + self.source_manager.clone() + } + /// Parse the given source file into a vector of top-level [Form]s. /// /// This does not run semantic analysis, or construct a [Module] from the parsed /// forms, and is largely intended for low-level testing of the parser. #[track_caller] - pub fn parse_forms(&mut self, source: Arc) -> Result, Report> { + pub fn parse_forms(&self, source: Arc) -> Result, Report> { crate::parser::parse_forms(source.clone()) .map_err(|err| Report::new(err).with_source_code(source)) } @@ -222,11 +232,14 @@ impl TestContext { /// This runs semantic analysis, and the returned module is guaranteed to be syntactically /// valid. #[track_caller] - pub fn parse_program(&mut self, source: impl Compile) -> Result, Report> { - source.compile_with_options(CompileOptions { - warnings_as_errors: self.assembler.warnings_as_errors(), - ..Default::default() - }) + pub fn parse_program(&self, source: impl Compile) -> Result, Report> { + source.compile_with_options( + self.source_manager.as_ref(), + CompileOptions { + warnings_as_errors: self.assembler.warnings_as_errors(), + ..Default::default() + }, + ) } /// Parse the given source file into a kernel [Module]. @@ -235,11 +248,14 @@ impl TestContext { /// valid. #[allow(unused)] #[track_caller] - pub fn parse_kernel(&mut self, source: impl Compile) -> Result, Report> { - source.compile_with_options(CompileOptions { - warnings_as_errors: self.assembler.warnings_as_errors(), - ..CompileOptions::for_kernel() - }) + pub fn parse_kernel(&self, source: impl Compile) -> Result, Report> { + source.compile_with_options( + self.source_manager.as_ref(), + CompileOptions { + warnings_as_errors: self.assembler.warnings_as_errors(), + ..CompileOptions::for_kernel() + }, + ) } /// Parse the given source file into an anonymous library [Module]. @@ -247,24 +263,30 @@ impl TestContext { /// This runs semantic analysis, and the returned module is guaranteed to be syntactically /// valid. #[track_caller] - pub fn parse_module(&mut self, source: impl Compile) -> Result, Report> { - source.compile_with_options(CompileOptions { - warnings_as_errors: self.assembler.warnings_as_errors(), - ..CompileOptions::for_library() - }) + pub fn parse_module(&self, source: impl Compile) -> Result, Report> { + source.compile_with_options( + self.source_manager.as_ref(), + CompileOptions { + warnings_as_errors: self.assembler.warnings_as_errors(), + ..CompileOptions::for_library() + }, + ) } /// Parse the given source file into a library [Module] with the given fully-qualified path. #[track_caller] pub fn parse_module_with_path( - &mut self, + &self, path: LibraryPath, source: impl Compile, ) -> Result, Report> { - source.compile_with_options(CompileOptions { - warnings_as_errors: self.assembler.warnings_as_errors(), - ..CompileOptions::new(ModuleKind::Library, path).unwrap() - }) + source.compile_with_options( + self.source_manager.as_ref(), + CompileOptions { + warnings_as_errors: self.assembler.warnings_as_errors(), + ..CompileOptions::new(ModuleKind::Library, path).unwrap() + }, + ) } /// Add `module` to the [Assembler] constructed by this context, making it available to @@ -305,15 +327,27 @@ impl TestContext { /// NOTE: Any modules added by, e.g. `add_module`, will be available to the executable /// module represented in `source`. #[track_caller] - pub fn assemble(&mut self, source: impl Compile) -> Result { + pub fn assemble(&self, source: impl Compile) -> Result { self.assembler.clone().assemble_program(source) } + /// Compile a [CompiledLibrary] from `modules` using the [Assembler] constructed by this + /// context. + /// + /// NOTE: Any modules added by, e.g. `add_module`, will be available to the library + #[track_caller] + pub fn assemble_library( + &self, + modules: impl IntoIterator>, + ) -> Result { + self.assembler.clone().assemble_library(modules) + } + /// Compile a module from `source`, with the fully-qualified name `path`, to MAST, returning /// the MAST roots of all the exported procedures of that module. #[track_caller] pub fn assemble_module( - &mut self, + &self, _path: LibraryPath, _module: impl Compile, ) -> Result, Report> { diff --git a/assembly/src/tests.rs b/assembly/src/tests.rs index 57af5bd3da..4a4440f04c 100644 --- a/assembly/src/tests.rs +++ b/assembly/src/tests.rs @@ -34,8 +34,8 @@ macro_rules! assert_assembler_diagnostic { #[test] fn simple_instructions() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin push.0 assertz end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin push.0 assertz end"); let program = context.assemble(source)?; let expected = "\ begin @@ -43,7 +43,7 @@ begin end"; assert_str_eq!(format!("{program}"), expected); - let source = source_file!("begin push.10 push.50 push.2 u32wrapping_madd end"); + let source = source_file!(&context, "begin push.10 push.50 push.2 u32wrapping_madd end"); let program = context.assemble(source)?; let expected = "\ begin @@ -51,7 +51,7 @@ begin end"; assert_str_eq!(format!("{program}"), expected); - let source = source_file!("begin push.10 push.50 push.2 u32wrapping_add3 end"); + let source = source_file!(&context, "begin push.10 push.50 push.2 u32wrapping_add3 end"); let program = context.assemble(source)?; let expected = "\ begin @@ -65,8 +65,8 @@ end"; #[test] #[ignore] fn empty_program() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin end"); let program = context.assemble(source)?; let expected = "begin basic_block noop end end"; assert_eq!(expected, format!("{}", program)); @@ -75,8 +75,8 @@ fn empty_program() -> TestResult { #[test] fn empty_if() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin if.true end end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin if.true end end"); assert_assembler_diagnostic!( context, source, @@ -94,8 +94,8 @@ fn empty_if() -> TestResult { #[test] fn empty_if_true_then_branch() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin if.true nop end end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin if.true nop end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -113,8 +113,8 @@ end"; #[test] #[ignore] fn empty_while() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin while.true end end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin while.true end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -130,8 +130,8 @@ end"; #[test] #[ignore] fn empty_repeat() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin repeat.5 end end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin repeat.5 end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -143,8 +143,8 @@ end"; #[test] fn single_basic_block() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin push.1 push.2 add end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin push.1 push.2 add end"); let program = context.assemble(source)?; let expected = "\ begin @@ -156,10 +156,10 @@ end"; #[test] fn basic_block_and_simple_if_true() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // if with else - let source = source_file!("begin push.2 push.3 if.true add else mul end end"); + let source = source_file!(&context, "begin push.2 push.3 if.true add else mul end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -175,7 +175,7 @@ end"; assert_str_eq!(format!("{program}"), expected); // if without else - let source = source_file!("begin push.2 push.3 if.true add end end"); + let source = source_file!(&context, "begin push.2 push.3 if.true add end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -194,10 +194,10 @@ end"; #[test] fn basic_block_and_simple_if_false() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // if with else - let source = source_file!("begin push.2 push.3 if.false add else mul end end"); + let source = source_file!(&context, "begin push.2 push.3 if.false add else mul end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -213,7 +213,7 @@ end"; assert_str_eq!(format!("{program}"), expected); // if without else - let source = source_file!("begin push.2 push.3 if.false add end end"); + let source = source_file!(&context, "begin push.2 push.3 if.false add end end"); let program = context.assemble(source)?; let expected = "\ begin @@ -242,6 +242,7 @@ fn simple_main_call() -> TestResult { let account_code = context.parse_module_with_path( account_path, source_file!( + &context, "\ export.account_method_1 push.2.1 add @@ -258,6 +259,7 @@ fn simple_main_call() -> TestResult { // compile note 1 program context.assemble(source_file!( + &context, " use.context::account begin @@ -268,6 +270,7 @@ fn simple_main_call() -> TestResult { // compile note 2 program context.assemble(source_file!( + &context, " use.context::account begin @@ -282,12 +285,13 @@ fn simple_main_call() -> TestResult { #[ignore] #[test] fn call_without_path() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // compile first module context.assemble_module( "account_code1".parse().unwrap(), source_file!( + &context, "\ export.account_method_1 push.2.1 add @@ -306,6 +310,7 @@ fn call_without_path() -> TestResult { context.assemble_module( "account_code2".parse().unwrap(), source_file!( + &context, "\ export.account_method_1 push.2.2 add @@ -322,6 +327,7 @@ fn call_without_path() -> TestResult { // compile program in which functions from different modules but with equal names are called context.assemble(source_file!( + &context, " begin # call the account_method_1 from the first module (account_code1) @@ -351,6 +357,7 @@ fn procref_call() -> TestResult { context.add_module_from_source( "module::path::one".parse().unwrap(), source_file!( + &context, " export.aaa push.7.8 @@ -366,6 +373,7 @@ fn procref_call() -> TestResult { context.add_module_from_source( "module::path::two".parse().unwrap(), source_file!( + &context, " use.module::path::one export.one::foo @@ -378,6 +386,7 @@ fn procref_call() -> TestResult { // compile program with procref calls context.assemble(source_file!( + &context, " use.module::path::two @@ -396,11 +405,12 @@ fn procref_call() -> TestResult { #[test] fn get_proc_name_of_unknown_module() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // Module `two` is unknown. This program should return // `AssemblyError::UndefinedProcedure`, referencing the // use of `bar` let module_source1 = source_file!( + &context, " use.module::path::two @@ -411,7 +421,7 @@ fn get_proc_name_of_unknown_module() -> TestResult { let module_path_one = "module::path::one".parse().unwrap(); let module1 = context.parse_module_with_path(module_path_one, module_source1)?; - let report = Assembler::default() + let report = Assembler::new(context.source_manager()) .assemble_library(core::iter::once(module1)) .expect_err("expected unknown module error"); @@ -434,8 +444,9 @@ fn get_proc_name_of_unknown_module() -> TestResult { #[test] fn simple_constant() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "\ const.TEST_CONSTANT=7 begin @@ -453,8 +464,9 @@ end"; #[test] fn multiple_constants_push() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.CONSTANT_1=21 \ const.CONSTANT_2=44 \ begin \ @@ -472,8 +484,9 @@ end"; #[test] fn constant_numeric_expression() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.TEST_CONSTANT=11-2+4*(12-(10+1))+9+8//4*2 \ begin \ push.TEST_CONSTANT \ @@ -491,8 +504,9 @@ end"; #[test] fn constant_alphanumeric_expression() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.TEST_CONSTANT_1=(18-1+10)*6-((13+7)*2) \ const.TEST_CONSTANT_2=11-2+4*(12-(10+1))+9 const.TEST_CONSTANT_3=(TEST_CONSTANT_1-(TEST_CONSTANT_2+10))//5+3 @@ -512,8 +526,9 @@ end"; #[test] fn constant_hexadecimal_value() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.TEST_CONSTANT=0xFF \ begin \ push.TEST_CONSTANT \ @@ -531,8 +546,9 @@ end"; #[test] fn constant_field_division() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.TEST_CONSTANT=(17//4)/4*(1//2)+2 \ begin \ push.TEST_CONSTANT \ @@ -550,8 +566,9 @@ end"; #[test] fn constant_err_const_not_initialized() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.TEST_CONSTANT=5+A \ begin \ push.TEST_CONSTANT \ @@ -573,8 +590,9 @@ fn constant_err_const_not_initialized() -> TestResult { #[test] fn constant_err_div_by_zero() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.TEST_CONSTANT=5/0 \ begin \ push.TEST_CONSTANT \ @@ -591,6 +609,7 @@ fn constant_err_div_by_zero() -> TestResult { ); let source = source_file!( + &context, "const.TEST_CONSTANT=5//0 \ begin \ push.TEST_CONSTANT \ @@ -610,8 +629,9 @@ fn constant_err_div_by_zero() -> TestResult { #[test] fn constants_must_be_uppercase() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.constant_1=12 \ begin \ push.constant_1 \ @@ -634,8 +654,9 @@ fn constants_must_be_uppercase() -> TestResult { #[test] fn duplicate_constant_name() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.CONSTANT=12 \ const.CONSTANT=14 \ begin \ @@ -661,8 +682,9 @@ fn duplicate_constant_name() -> TestResult { #[test] fn constant_must_be_valid_felt() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.CONSTANT=1122INVALID \ begin \ push.CONSTANT \ @@ -686,8 +708,9 @@ or \"export\", or \"proc\", or \"use\", or end of file, or doc comment" #[test] fn constant_must_be_within_valid_felt_range() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "const.CONSTANT=18446744073709551615 \ begin \ push.CONSTANT \ @@ -708,8 +731,9 @@ fn constant_must_be_within_valid_felt_range() -> TestResult { #[test] fn constants_defined_in_global_scope() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, " begin \ const.CONSTANT=12 @@ -735,8 +759,9 @@ fn constants_defined_in_global_scope() -> TestResult { #[test] fn constant_not_found() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::new(); let source = source_file!( + &context, " begin \ push.CONSTANT \ @@ -760,7 +785,7 @@ fn constant_not_found() -> TestResult { #[test] fn mem_operations_with_constants() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // Define constant values const PROC_LOC_STORE_PTR: u64 = 0; @@ -772,8 +797,10 @@ fn mem_operations_with_constants() -> TestResult { const GLOBAL_STOREW_PTR: u64 = 6; const GLOBAL_LOADW_PTR: u64 = 7; - let source = source_file!(format!( - "\ + let source = source_file!( + &context, + format!( + "\ const.PROC_LOC_STORE_PTR={PROC_LOC_STORE_PTR} const.PROC_LOC_LOAD_PTR={PROC_LOC_LOAD_PTR} const.PROC_LOC_STOREW_PTR={PROC_LOC_STOREW_PTR} @@ -817,12 +844,15 @@ fn mem_operations_with_constants() -> TestResult { mem_loadw.GLOBAL_LOADW_PTR end " - )); + ) + ); let program = context.assemble(source)?; // Define expected - let expected = source_file!(format!( - "\ + let expected = source_file!( + &context, + format!( + "\ proc.test_const_loc.4 # constant should resolve using locaddr operation locaddr.{PROC_LOC_STORE_PTR} @@ -857,7 +887,8 @@ fn mem_operations_with_constants() -> TestResult { mem_loadw.{GLOBAL_LOADW_PTR} end " - )); + ) + ); let expected_program = context.assemble(expected)?; assert_str_eq!(expected_program.to_string(), program.to_string()); Ok(()) @@ -868,8 +899,11 @@ fn const_conversion_failed_to_u16() -> TestResult { // Define constant value greater than u16::MAX let constant_value: u64 = u16::MAX as u64 + 1; - let source = source_file!(format!( - "\ + let context = TestContext::default(); + let source = source_file!( + &context, + format!( + "\ const.CONSTANT={constant_value} proc.test_constant_overflow.1 @@ -880,8 +914,8 @@ fn const_conversion_failed_to_u16() -> TestResult { exec.test_constant_overflow end " - )); - let mut context = TestContext::default(); + ) + ); assert_assembler_diagnostic!( context, @@ -901,19 +935,22 @@ fn const_conversion_failed_to_u16() -> TestResult { #[test] fn const_conversion_failed_to_u32() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // Define constant value greater than u16::MAX let constant_value: u64 = u32::MAX as u64 + 1; - let source = source_file!(format!( - "\ + let source = source_file!( + &context, + format!( + "\ const.CONSTANT={constant_value} begin mem_load.CONSTANT end " - )); + ) + ); assert_assembler_diagnostic!( context, @@ -936,7 +973,9 @@ fn const_conversion_failed_to_u32() -> TestResult { #[test] fn assert_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -947,7 +986,6 @@ fn assert_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -960,7 +998,9 @@ end"; #[test] fn assertz_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -971,7 +1011,6 @@ fn assertz_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -984,7 +1023,9 @@ end"; #[test] fn assert_eq_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -995,7 +1036,6 @@ fn assert_eq_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -1008,7 +1048,9 @@ end"; #[test] fn assert_eqw_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -1019,7 +1061,6 @@ fn assert_eqw_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -1066,7 +1107,9 @@ end"; #[test] fn u32assert_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -1077,7 +1120,6 @@ fn u32assert_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -1100,7 +1142,9 @@ end"; #[test] fn u32assert2_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -1111,7 +1155,6 @@ fn u32assert2_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -1124,7 +1167,9 @@ end"; #[test] fn u32assertw_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -1135,7 +1180,6 @@ fn u32assertw_with_code() -> TestResult { end " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -1167,7 +1211,9 @@ end"; #[test] fn mtree_verify_with_code() -> TestResult { + let context = TestContext::default(); let source = source_file!( + &context, "\ const.ERR1=1 @@ -1179,7 +1225,6 @@ fn mtree_verify_with_code() -> TestResult { " ); - let mut context = TestContext::default(); let program = context.assemble(source)?; let expected = "\ @@ -1195,10 +1240,11 @@ end"; #[test] fn nested_control_blocks() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // if with else let source = source_file!( + &context, "begin \ push.2 push.3 \ if.true \ @@ -1251,9 +1297,11 @@ end"; #[test] fn program_with_one_procedure() -> TestResult { - let mut context = TestContext::default(); - let source = - source_file!("proc.foo push.3 push.7 mul end begin push.2 push.3 add exec.foo end"); + let context = TestContext::default(); + let source = source_file!( + &context, + "proc.foo push.3 push.7 mul end begin push.2 push.3 add exec.foo end" + ); let program = context.assemble(source)?; let expected = "\ begin @@ -1268,8 +1316,9 @@ end"; #[test] fn program_with_nested_procedure() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "\ proc.foo push.3 push.7 mul end \ proc.bar push.5 exec.foo add end \ @@ -1304,8 +1353,9 @@ end"; #[test] fn program_with_proc_locals() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "\ proc.foo.1 \ loc_store.0 \ @@ -1346,8 +1396,9 @@ end"; #[test] fn program_with_proc_locals_fail() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "\ proc.foo \ loc_store.0 \ @@ -1371,9 +1422,11 @@ fn program_with_proc_locals_fail() -> TestResult { #[test] fn program_with_exported_procedure() -> TestResult { - let mut context = TestContext::default(); - let source = - source_file!("export.foo push.3 push.7 mul end begin push.2 push.3 add exec.foo end"); + let context = TestContext::default(); + let source = source_file!( + &context, + "export.foo push.3 push.7 mul end begin push.2 push.3 add exec.foo end" + ); assert_assembler_diagnostic!( context, @@ -1394,8 +1447,8 @@ fn program_with_exported_procedure() -> TestResult { #[test] fn program_with_dynamic_code_execution() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin dynexec end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin dynexec end"); let program = context.assemble(source)?; let expected = "\ begin @@ -1407,8 +1460,8 @@ end"; #[test] fn program_with_dynamic_code_execution_in_new_context() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin dyncall end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin dyncall end"); let program = context.assemble(source)?; let expected = "\ begin @@ -1423,8 +1476,8 @@ end"; #[test] fn program_with_incorrect_mast_root_length() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin call.0x1234 end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin call.0x1234 end"); assert_assembler_diagnostic!( context, @@ -1440,8 +1493,9 @@ fn program_with_incorrect_mast_root_length() -> TestResult { #[test] fn program_with_invalid_mast_root_chars() { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "begin call.0xc2545da99d3a1f3f38d957c7893c44d78998d8ea8b11aba7e22c8c2b2a21xyzb end" ); @@ -1458,8 +1512,9 @@ fn program_with_invalid_mast_root_chars() { #[test] fn program_with_invalid_rpo_digest_call() { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "begin call.0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff end" ); @@ -1476,13 +1531,14 @@ fn program_with_invalid_rpo_digest_call() { #[test] fn program_with_phantom_mast_call() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); let source = source_file!( + &context, "begin call.0xc2545da99d3a1f3f38d957c7893c44d78998d8ea8b11aba7e22c8c2b2a213dae end" ); let ast = context.parse_program(source)?; - let assembler = Assembler::default().with_debug_mode(true); + let assembler = Assembler::new(context.source_manager()).with_debug_mode(true); assembler.assemble_program(ast)?; Ok(()) } @@ -1505,20 +1561,26 @@ fn program_with_one_import_and_hex_call() -> TestResult { let mut context = TestContext::default(); let path = MODULE.parse().unwrap(); - let ast = context.parse_module_with_path(path, source_file!(PROCEDURE.to_string()))?; - let library = Assembler::default().assemble_library(core::iter::once(ast)).unwrap(); + let ast = + context.parse_module_with_path(path, source_file!(&context, PROCEDURE.to_string()))?; + let library = Assembler::new(context.source_manager()) + .assemble_library(core::iter::once(ast)) + .unwrap(); context.add_library(&library)?; - let source = source_file!(format!( - r#" + let source = source_file!( + &context, + format!( + r#" use.{MODULE} begin push.4 push.3 exec.u256::iszero_unsafe call.0x20234ee941e53a15886e733cc8e041198c6e90d2a16ea18ce1030e8c3596dd38 end"# - )); + ) + ); let program = context.assemble(source)?; let expected = "\ @@ -1559,20 +1621,26 @@ fn program_with_two_imported_procs_with_same_mast_root() -> TestResult { let mut context = TestContext::default(); let path = MODULE.parse().unwrap(); - let ast = context.parse_module_with_path(path, source_file!(PROCEDURE.to_string()))?; - let library = Assembler::default().assemble_library(core::iter::once(ast)).unwrap(); + let ast = + context.parse_module_with_path(path, source_file!(&context, PROCEDURE.to_string()))?; + let library = Assembler::new(context.source_manager()) + .assemble_library(core::iter::once(ast)) + .unwrap(); context.add_library(&library)?; - let source = source_file!(format!( - r#" + let source = source_file!( + &context, + format!( + r#" use.{MODULE} begin push.4 push.3 exec.u256::iszero_unsafe exec.u256::iszero_unsafe_dup end"# - )); + ) + ); context.assemble(source)?; Ok(()) } @@ -1609,7 +1677,10 @@ fn program_with_reexported_proc_in_same_library() -> TestResult { "#; let mut context = TestContext::new(); - let ast = Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, MODULE_BODY).unwrap(); + let mut parser = Module::parser(ModuleKind::Library); + let ast = parser + .parse_str(MODULE.parse().unwrap(), MODULE_BODY, &context.source_manager()) + .unwrap(); // check docs let docs_checked_eqz = @@ -1625,23 +1696,29 @@ fn program_with_reexported_proc_in_same_library() -> TestResult { "unchecked_eqz checks if the value is zero and returns 1 if it is, 0 otherwise\n" ); - let ref_ast = - Module::parse_str(REF_MODULE.parse().unwrap(), ModuleKind::Library, REF_MODULE_BODY) - .unwrap(); + let mut parser = Module::parser(ModuleKind::Library); + let ref_ast = parser + .parse_str(REF_MODULE.parse().unwrap(), REF_MODULE_BODY, &context.source_manager()) + .unwrap(); - let library = Assembler::default().assemble_library([ast, ref_ast]).unwrap(); + let library = Assembler::new(context.source_manager()) + .assemble_library([ast, ref_ast]) + .unwrap(); context.add_library(&library)?; - let source = source_file!(format!( - r#" + let source = source_file!( + &context, + format!( + r#" use.{MODULE} begin push.4 push.3 exec.u256::checked_eqz exec.u256::notchecked_eqz end"# - )); + ) + ); let program = context.assemble(source)?; let expected = "\ begin @@ -1685,26 +1762,32 @@ fn program_with_reexported_proc_in_another_library() -> TestResult { "#; let mut context = TestContext::default(); + let mut parser = Module::parser(ModuleKind::Library); + let source_manager = context.source_manager(); // We reference code in this module let ref_ast = - Module::parse_str(REF_MODULE.parse().unwrap(), ModuleKind::Library, REF_MODULE_BODY)?; + parser.parse_str(REF_MODULE.parse().unwrap(), REF_MODULE_BODY, &source_manager)?; // But only exports from this module are exposed by the library - let ast = Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, MODULE_BODY)?; + let ast = parser.parse_str(MODULE.parse().unwrap(), MODULE_BODY, &source_manager)?; - let dummy_library = Assembler::default().with_module(ref_ast)?.assemble_library([ast])?; + let dummy_library = + Assembler::new(source_manager).with_module(ref_ast)?.assemble_library([ast])?; // Now we want to use the the library we've compiled context.add_library(&dummy_library)?; - let source = source_file!(format!( - r#" + let source = source_file!( + &context, + format!( + r#" use.{MODULE} begin push.4 push.3 exec.u256::checked_eqz exec.u256::notchecked_eqz end"# - )); + ) + ); let program = context.assemble(source)?; let expected = "\ @@ -1723,15 +1806,18 @@ end"; let mut context = TestContext::default(); context.add_library(dummy_library)?; - let source = source_file!(format!( - r#" + let source = source_file!( + &context, + format!( + r#" use.{REF_MODULE} begin push.4 push.3 exec.u64::checked_eqz exec.u64::notchecked_eqz end"# - )); + ) + ); assert_assembler_diagnostic!(context, source, "undefined module 'dummy2::math::u64'"); Ok(()) } @@ -1754,12 +1840,15 @@ fn module_alias() -> TestResult { end"#; let mut context = TestContext::default(); - let ast = Module::parse_str(MODULE.parse().unwrap(), ModuleKind::Library, PROCEDURE).unwrap(); - let library = Assembler::default().assemble_library([ast]).unwrap(); + let source_manager = context.source_manager(); + let mut parser = Module::parser(ModuleKind::Library); + let ast = parser.parse_str(MODULE.parse().unwrap(), PROCEDURE, &source_manager).unwrap(); + let library = Assembler::new(source_manager).assemble_library([ast]).unwrap(); context.add_library(&library)?; let source = source_file!( + &context, " use.dummy::math::u64->bigint @@ -1782,6 +1871,7 @@ end"; // --- invalid module alias ----------------------------------------------- let source = source_file!( + &context, " use.dummy::math::u64->bigint->invalidname @@ -1807,6 +1897,7 @@ end"; // --- duplicate module import -------------------------------------------- let source = source_file!( + &context, " use.dummy::math::u64 use.dummy::math::u64->bigint @@ -1840,6 +1931,7 @@ end"; // fail for the time being /* let source = source_file!( + &context, " use.dummy::math::u64->bigint use.dummy::math::u64->bigint2 @@ -1857,9 +1949,10 @@ end"; #[test] fn program_with_import_errors() { - let mut context = TestContext::default(); + let context = TestContext::default(); // --- non-existent import ------------------------------------------------ let source = source_file!( + &context, "\ use.std::math::u512 begin \ @@ -1881,6 +1974,7 @@ fn program_with_import_errors() { // --- non-existent procedure in import ----------------------------------- let source = source_file!( + &context, "\ use.std::math::u256 begin \ @@ -1906,8 +2000,8 @@ fn program_with_import_errors() { #[test] fn comment_simple() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin # simple comment \n push.1 push.2 add end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin # simple comment \n push.1 push.2 add end"); let program = context.assemble(source)?; let expected = "\ begin @@ -1919,10 +2013,11 @@ end"; #[test] fn comment_in_nested_control_blocks() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // if with else let source = source_file!( + &context, "begin \ push.1 push.2 \ if.true \ @@ -1974,8 +2069,8 @@ end"; #[test] fn comment_before_program() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!(" # starting comment \n begin push.1 push.2 add end"); + let context = TestContext::default(); + let source = source_file!(&context, " # starting comment \n begin push.1 push.2 add end"); let program = context.assemble(source)?; let expected = "\ begin @@ -1987,8 +2082,8 @@ end"; #[test] fn comment_after_program() -> TestResult { - let mut context = TestContext::default(); - let source = source_file!("begin push.1 push.2 add end # closing comment"); + let context = TestContext::default(); + let source = source_file!(&context, "begin push.1 push.2 add end # closing comment"); let program = context.assemble(source)?; let expected = "\ begin @@ -2003,10 +2098,10 @@ end"; #[test] fn invalid_empty_program() { - let mut context = TestContext::default(); + let context = TestContext::default(); assert_assembler_diagnostic!( context, - source_file!(""), + source_file!(&context, ""), "unexpected end of file", regex!(r#",-\[test[\d]+:1:1\]"#), "`----", @@ -2015,7 +2110,7 @@ fn invalid_empty_program() { assert_assembler_diagnostic!( context, - source_file!(""), + source_file!(&context, ""), "unexpected end of file", regex!(r#",-\[test[\d]+:1:1\]"#), " `----", @@ -2025,10 +2120,10 @@ fn invalid_empty_program() { #[test] fn invalid_program_unrecognized_token() { - let mut context = TestContext::default(); + let context = TestContext::default(); assert_assembler_diagnostic!( context, - source_file!("none"), + source_file!(&context, "none"), "invalid syntax", regex!(r#",-\[test[\d]+:1:1\]"#), "1 | none", @@ -2041,10 +2136,10 @@ fn invalid_program_unrecognized_token() { #[test] fn invalid_program_unmatched_begin() { - let mut context = TestContext::default(); + let context = TestContext::default(); assert_assembler_diagnostic!( context, - source_file!("begin add"), + source_file!(&context, "begin add"), "unexpected end of file", regex!(r#",-\[test[\d]+:1:10\]"#), "1 | begin add", @@ -2055,10 +2150,10 @@ fn invalid_program_unmatched_begin() { #[test] fn invalid_program_invalid_top_level_token() { - let mut context = TestContext::default(); + let context = TestContext::default(); assert_assembler_diagnostic!( context, - source_file!("begin add end mul"), + source_file!(&context, "begin add end mul"), "invalid syntax", regex!(r#",-\[test[\d]+:1:15\]"#), "1 | begin add end mul", @@ -2071,8 +2166,8 @@ fn invalid_program_invalid_top_level_token() { #[test] fn invalid_proc_missing_end_unexpected_begin() { - let mut context = TestContext::default(); - let source = source_file!("proc.foo add mul begin push.1 end"); + let context = TestContext::default(); + let source = source_file!(&context, "proc.foo add mul begin push.1 end"); assert_assembler_diagnostic!( context, source, @@ -2088,8 +2183,8 @@ fn invalid_proc_missing_end_unexpected_begin() { #[test] fn invalid_proc_missing_end_unexpected_proc() { - let mut context = TestContext::default(); - let source = source_file!("proc.foo add mul proc.bar push.3 end begin push.1 end"); + let context = TestContext::default(); + let source = source_file!(&context, "proc.foo add mul proc.bar push.3 end begin push.1 end"); assert_assembler_diagnostic!( context, source, @@ -2105,8 +2200,8 @@ fn invalid_proc_missing_end_unexpected_proc() { #[test] fn invalid_proc_undefined_local() { - let mut context = TestContext::default(); - let source = source_file!("proc.foo add mul end begin push.1 exec.bar end"); + let context = TestContext::default(); + let source = source_file!(&context, "proc.foo add mul end begin push.1 exec.bar end"); assert_assembler_diagnostic!( context, source, @@ -2123,8 +2218,8 @@ fn invalid_proc_undefined_local() { #[test] fn invalid_proc_invalid_numeric_name() { - let mut context = TestContext::default(); - let source = source_file!("proc.123 add mul end begin push.1 exec.123 end"); + let context = TestContext::default(); + let source = source_file!(&context, "proc.123 add mul end begin push.1 exec.123 end"); assert_assembler_diagnostic!( context, source, @@ -2141,8 +2236,9 @@ fn invalid_proc_invalid_numeric_name() { #[test] fn invalid_proc_duplicate_procedure_name() { - let mut context = TestContext::default(); - let source = source_file!("proc.foo add mul end proc.foo push.3 end begin push.1 end"); + let context = TestContext::default(); + let source = + source_file!(&context, "proc.foo add mul end proc.foo push.3 end begin push.1 end"); assert_assembler_diagnostic!( context, source, @@ -2160,8 +2256,8 @@ fn invalid_proc_duplicate_procedure_name() { #[test] fn invalid_if_missing_end_no_else() { - let mut context = TestContext::default(); - let source = source_file!("begin push.1 add if.true mul"); + let context = TestContext::default(); + let source = source_file!(&context, "begin push.1 add if.true mul"); assert_assembler_diagnostic!( context, source, @@ -2175,8 +2271,8 @@ fn invalid_if_missing_end_no_else() { #[test] fn invalid_else_with_no_if() { - let mut context = TestContext::default(); - let source = source_file!("begin push.1 add else mul end"); + let context = TestContext::default(); + let source = source_file!(&context, "begin push.1 add else mul end"); assert_assembler_diagnostic!( context, source, @@ -2189,7 +2285,7 @@ fn invalid_else_with_no_if() { r#" help: expected primitive opcode (e.g. "add"), or "end", or control flow opcode (e.g. "if.true")"# ); - let source = source_file!("begin push.1 while.true add else mul end end"); + let source = source_file!(&context, "begin push.1 while.true add else mul end end"); assert_assembler_diagnostic!( context, source, @@ -2205,9 +2301,10 @@ fn invalid_else_with_no_if() { #[test] fn invalid_unmatched_else_within_if_else() { - let mut context = TestContext::default(); + let context = TestContext::default(); - let source = source_file!("begin push.1 if.true add else mul else push.1 end end end"); + let source = + source_file!(&context, "begin push.1 if.true add else mul else push.1 end end end"); assert_assembler_diagnostic!( context, source, @@ -2223,9 +2320,9 @@ fn invalid_unmatched_else_within_if_else() { #[test] fn invalid_if_else_no_matching_end() { - let mut context = TestContext::default(); + let context = TestContext::default(); - let source = source_file!("begin push.1 add if.true mul else add"); + let source = source_file!(&context, "begin push.1 add if.true mul else add"); assert_assembler_diagnostic!( context, source, @@ -2239,10 +2336,10 @@ fn invalid_if_else_no_matching_end() { #[test] fn invalid_repeat() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); // unmatched repeat - let source = source_file!("begin push.1 add repeat.10 mul"); + let source = source_file!(&context, "begin push.1 add repeat.10 mul"); assert_assembler_diagnostic!( context, source, @@ -2254,7 +2351,7 @@ fn invalid_repeat() -> TestResult { ); // invalid iter count - let source = source_file!("begin push.1 add repeat.23x3 mul end end"); + let source = source_file!(&context, "begin push.1 add repeat.23x3 mul end end"); assert_assembler_diagnostic!( context, source, @@ -2272,9 +2369,9 @@ fn invalid_repeat() -> TestResult { #[test] fn invalid_while() -> TestResult { - let mut context = TestContext::default(); + let context = TestContext::default(); - let source = source_file!("begin push.1 add while mul end end"); + let source = source_file!(&context, "begin push.1 add while mul end end"); assert_assembler_diagnostic!( context, source, @@ -2287,7 +2384,7 @@ fn invalid_while() -> TestResult { r#" help: expected ".""# ); - let source = source_file!("begin push.1 add while.abc mul end end"); + let source = source_file!(&context, "begin push.1 add while.abc mul end end"); assert_assembler_diagnostic!( context, source, @@ -2300,7 +2397,7 @@ fn invalid_while() -> TestResult { r#" help: expected "true""# ); - let source = source_file!("begin push.1 add while.true mul"); + let source = source_file!(&context, "begin push.1 add while.true mul"); assert_assembler_diagnostic!( context, source, @@ -2317,9 +2414,11 @@ fn invalid_while() -> TestResult { // ================================================================================================ #[test] fn test_compiled_library() { + let context = TestContext::new(); let mut mod_parser = ModuleParser::new(ModuleKind::Library); let mod1 = { let source = source_file!( + &context, " proc.internal push.5 @@ -2339,6 +2438,7 @@ fn test_compiled_library() { let mod2 = { let source = source_file!( + &context, " export.foo push.7 @@ -2355,14 +2455,14 @@ fn test_compiled_library() { }; let compiled_library = { - let assembler = Assembler::default(); + let assembler = Assembler::new(context.source_manager()); assembler.assemble_library([mod1, mod2]).unwrap() }; assert_eq!(compiled_library.exports().count(), 4); // Compile program that uses compiled library - let mut assembler = Assembler::default(); + let mut assembler = Assembler::new(context.source_manager()); assembler.add_compiled_library(&compiled_library).unwrap(); diff --git a/core/Cargo.toml b/core/Cargo.toml index 1ec70adaef..90ffa79991 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -19,18 +19,28 @@ doctest = false [features] default = ["std"] +diagnostics = ["dep:miette"] std = [ - "miden-crypto/std", - "miden-formatting/std", - "math/std", - "winter-utils/std", - "thiserror/std", + "dep:parking_lot", + "memchr/std", + "miden-crypto/std", + "miden-formatting/std", + "math/std", + "winter-utils/std", + "thiserror/std", ] [dependencies] math = { package = "winter-math", version = "0.9", default-features = false } +memchr = { version = "2.7", default-features = false } miden-crypto = { git = "https://github.com/0xPolygonMiden/crypto", branch = "next", default-features = false } miden-formatting = { version = "0.1", default-features = false } +miette = { version = "7.1.0", git = "https://github.com/bitwalker/miette", branch = "no-std", default-features = false, features = [ + "fancy-no-syscall", + "derive", +], optional = true } +lock_api = { version = "0.4", features = ["arc_lock"] } +parking_lot = { version = "0.12", optional = true } num-derive = { version = "0.4", default-features = false } num-traits = { version = "0.2", default-features = false } thiserror = { version = "1.0", git = "https://github.com/bitwalker/thiserror", branch = "no-std", default-features = false } @@ -39,3 +49,11 @@ winter-utils = { package = "winter-utils", version = "0.9", default-features = f [dev-dependencies] proptest = "1.3" rand_utils = { version = "0.9", package = "winter-rand-utils" } +loom = "0.7" + +[target.'cfg(loom)'.dependencies] +loom = "0.7" + +# Enable once the VM reaches Rust 1.80+ +#[lints.rust] +#unexpected_cfgs = { level = "warn", check-cfg = ['cfg(loom)'] } diff --git a/core/build.rs b/core/build.rs new file mode 100644 index 0000000000..107d736266 --- /dev/null +++ b/core/build.rs @@ -0,0 +1,4 @@ +fn main() { + // Needed until we have an MSRV of 1.80+ + println!("cargo::rustc-check-cfg=cfg(loom)"); +} diff --git a/core/src/debuginfo/location.rs b/core/src/debuginfo/location.rs new file mode 100644 index 0000000000..47f8132f64 --- /dev/null +++ b/core/src/debuginfo/location.rs @@ -0,0 +1,71 @@ +use alloc::sync::Arc; +use core::{fmt, ops::Range}; + +use super::ByteIndex; + +/// A [Location] represents file and span information for portability across source managers +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Location { + /// The path to the source file in which the relevant source code can be found + pub path: Arc, + /// The starting byte index (inclusive) of this location + pub start: ByteIndex, + /// The ending byte index (exclusive) of this location + pub end: ByteIndex, +} + +impl Location { + /// Creates a new [Location]. + pub const fn new(path: Arc, start: ByteIndex, end: ByteIndex) -> Self { + Self { path, start, end } + } + + /// Get the name (or path) of the source file + pub fn path(&self) -> Arc { + self.path.clone() + } + + /// Returns the byte range represented by this location + pub const fn range(&self) -> Range { + self.start..self.end + } +} + +/// A [FileLineCol] represents traditional file/line/column information for use in rendering. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FileLineCol { + /// The path to the source file in which the relevant source code can be found + pub path: Arc, + /// The one-indexed number of the line to which this location refers + pub line: u32, + /// The one-indexed column of the line on which this location starts + pub column: u32, +} + +impl FileLineCol { + /// Creates a new [Location]. + pub const fn new(path: Arc, line: u32, column: u32) -> Self { + Self { path, line, column } + } + + /// Get the name (or path) of the source file + pub fn path(&self) -> Arc { + self.path.clone() + } + + /// Returns the line of the location. + pub const fn line(&self) -> u32 { + self.line + } + + /// Moves the column by the given offset. + pub fn move_column(&mut self, offset: u32) { + self.column += offset; + } +} + +impl fmt::Display for FileLineCol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{}@{}:{}]", &self.path, self.line, self.column) + } +} diff --git a/core/src/debuginfo/mod.rs b/core/src/debuginfo/mod.rs new file mode 100644 index 0000000000..009f57f535 --- /dev/null +++ b/core/src/debuginfo/mod.rs @@ -0,0 +1,14 @@ +mod location; +mod source_file; +mod source_manager; +mod span; + +pub use self::location::{FileLineCol, Location}; +pub use self::source_file::{ + ByteIndex, ByteOffset, ColumnIndex, LineIndex, SourceContent, SourceFile, SourceFileRef, +}; +pub use self::source_manager::{DefaultSourceManager, SourceId, SourceManager}; +pub use self::span::{SourceSpan, Span, Spanned}; + +#[cfg(feature = "std")] +pub use self::source_manager::SourceManagerExt; diff --git a/core/src/debuginfo/source_file.rs b/core/src/debuginfo/source_file.rs new file mode 100644 index 0000000000..0ba7b95744 --- /dev/null +++ b/core/src/debuginfo/source_file.rs @@ -0,0 +1,774 @@ +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use core::{fmt, num::NonZeroU32, ops::Range}; + +use super::{FileLineCol, SourceId, SourceSpan}; + +// SOURCE FILE +// ================================================================================================ + +/// A [SourceFile] represents a single file stored in a [super::SourceManager] +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SourceFile { + /// The unique identifier allocated for this [SourceFile] by its owning [super::SourceManager] + id: SourceId, + /// The file content + content: SourceContent, +} + +#[cfg(feature = "diagnostics")] +impl miette::SourceCode for SourceFile { + fn read_span<'a>( + &'a self, + span: &miette::SourceSpan, + context_lines_before: usize, + context_lines_after: usize, + ) -> Result + 'a>, miette::MietteError> { + let mut start = + u32::try_from(span.offset()).map_err(|_| miette::MietteError::OutOfBounds)?; + let len = u32::try_from(span.len()).map_err(|_| miette::MietteError::OutOfBounds)?; + let mut end = start.checked_add(len).ok_or(miette::MietteError::OutOfBounds)?; + if context_lines_before > 0 { + let line_index = self.content.line_index(start.into()); + let start_line_index = line_index.saturating_sub(context_lines_before as u32); + start = self.content.line_start(start_line_index).map(|idx| idx.to_u32()).unwrap_or(0); + } + if context_lines_after > 0 { + let line_index = self.content.line_index(end.into()); + let end_line_index = line_index + .checked_add(context_lines_after as u32) + .ok_or(miette::MietteError::OutOfBounds)?; + end = self + .content + .line_range(end_line_index) + .map(|range| range.end.to_u32()) + .unwrap_or_else(|| self.content.source_range().end.to_u32()); + } + Ok(Box::new(ScopedSourceFileRef { + file: self, + span: miette::SourceSpan::new((start as usize).into(), end.abs_diff(start) as usize), + })) + } +} + +impl SourceFile { + /// Create a new [SourceFile] from its raw components + pub fn new(id: SourceId, path: impl Into>, content: impl Into>) -> Self { + let path = path.into(); + let content = SourceContent::new(path, content.into()); + Self { id, content } + } + + pub(super) fn from_raw_parts(id: SourceId, content: SourceContent) -> Self { + Self { id, content } + } + + /// Get the [SourceId] associated with this file + pub const fn id(&self) -> SourceId { + self.id + } + + /// Get the name of this source file + pub fn name(&self) -> Arc { + self.content.name() + } + + /// Get the path of this source file as a [std::path::Path] + #[cfg(feature = "std")] + #[inline] + pub fn path(&self) -> &std::path::Path { + self.content.path() + } + + /// Returns a reference to the underlying [SourceContent] + pub fn content(&self) -> &SourceContent { + &self.content + } + + /// Returns the number of lines in this file + pub fn line_count(&self) -> usize { + self.content.last_line_index().to_usize() + 1 + } + + /// Returns the number of bytes in this file + pub fn len(&self) -> usize { + self.content.len() + } + + /// Returns true if this file is empty + pub fn is_empty(&self) -> bool { + self.content.is_empty() + } + + /// Get the underlying content of this file + #[inline(always)] + pub fn as_str(&self) -> &str { + self.content.as_str() + } + + /// Get the underlying content of this file as a byte slice + #[inline(always)] + pub fn as_bytes(&self) -> &[u8] { + self.content.as_bytes() + } + + /// Returns a [SourceSpan] covering the entirety of this file + #[inline] + pub fn source_span(&self) -> SourceSpan { + let range = self.content.source_range(); + SourceSpan::new(self.id, range.start.0..range.end.0) + } + + /// Returns a subset of the underlying content as a string slice. + /// + /// The bounds of the given span are character indices, _not_ byte indices. + /// + /// Returns `None` if the given span is out of bounds + #[inline(always)] + pub fn source_slice(&self, span: impl Into>) -> Option<&str> { + self.content.source_slice(span) + } + + /// Returns a [SourceFileRef] corresponding to the bytes contained in the specified span. + pub fn slice(self: &Arc, span: impl Into>) -> SourceFileRef { + SourceFileRef::new(Arc::clone(self), span) + } + + /// Get a [SourceSpan] which points to the first byte of the character at `column` on `line` + /// + /// Returns `None` if the given line/column is out of bounds for this file. + pub fn line_column_to_span(&self, line: u32, column: u32) -> Option { + let line_index = LineIndex::from(line.saturating_sub(1)); + let column_index = ColumnIndex::from(column.saturating_sub(1)); + let offset = self.content.line_column_to_offset(line_index, column_index)?; + Some(SourceSpan::at(self.id, offset.0)) + } + + /// Get a [FileLineCol] equivalent to the start of the given [SourceSpan] + pub fn location(&self, span: SourceSpan) -> FileLineCol { + assert_eq!(span.source_id(), self.id, "mismatched source ids"); + + self.content + .location(ByteIndex(span.into_range().start)) + .expect("invalid source span: starting byte is out of bounds") + } +} + +impl AsRef for SourceFile { + #[inline(always)] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl AsRef<[u8]> for SourceFile { + #[inline(always)] + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +#[cfg(feature = "std")] +impl AsRef for SourceFile { + #[inline(always)] + fn as_ref(&self) -> &std::path::Path { + self.path() + } +} + +// SOURCE FILE REF +// ================================================================================================ + +/// A reference to a specific spanned region of a [SourceFile], that provides access to the actual +/// [SourceFile], but scoped to the span it was created with. +/// +/// This is useful in error types that implement [miette::Diagnostic], as it contains all of the +/// data necessary to render the source code being referenced, without a [super::SourceManager] on +/// hand. +#[derive(Debug, Clone)] +pub struct SourceFileRef { + file: Arc, + span: SourceSpan, +} + +impl SourceFileRef { + /// Create a [SourceFileRef] from a [SourceFile] and desired span (in bytes) + /// + /// The given span will be constrained to the bytes of `file`, so a span that reaches out of + /// bounds will have its end bound set to the last byte of the file. + pub fn new(file: Arc, span: impl Into>) -> Self { + let span = span.into(); + let end = core::cmp::min(span.end, file.len() as u32); + let span = SourceSpan::new(file.id(), span.start..end); + Self { file, span } + } + + /// Returns a ref-counted handle to the underlying [SourceFile] + pub fn source_file(&self) -> Arc { + self.file.clone() + } + + /// Returns the name of the file this [SourceFileRef] is selecting, as a [std::path::Path] + #[cfg(feature = "std")] + pub fn path(&self) -> &std::path::Path { + self.file.path() + } + + /// Returns the name of the file this [SourceFileRef] is selecting + pub fn name(&self) -> &str { + self.file.content.path.as_ref() + } + + /// Returns the [SourceSpan] selected by this [SourceFileRef] + pub const fn span(&self) -> SourceSpan { + self.span + } + + /// Returns the underlying `str` selected by this [SourceFileRef] + pub fn as_str(&self) -> &str { + self.file.source_slice(self.span).unwrap() + } + + /// Returns the underlying bytes selected by this [SourceFileRef] + #[inline] + pub fn as_bytes(&self) -> &[u8] { + self.as_str().as_bytes() + } + + /// Returns the number of bytes represented by the subset of the underlying file that is covered + /// by this [SourceFileRef] + pub fn len(&self) -> usize { + self.span.len() + } + + /// Returns true if this selection is empty + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Eq for SourceFileRef {} + +impl PartialEq for SourceFileRef { + fn eq(&self, other: &Self) -> bool { + self.as_str() == other.as_str() + } +} + +impl Ord for SourceFileRef { + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.as_str().cmp(other.as_str()) + } +} + +impl PartialOrd for SourceFileRef { + #[inline(always)] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl core::hash::Hash for SourceFileRef { + fn hash(&self, state: &mut H) { + self.span.hash(state); + self.as_str().hash(state); + } +} + +impl AsRef for SourceFileRef { + #[inline(always)] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl AsRef<[u8]> for SourceFileRef { + #[inline(always)] + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +#[cfg(feature = "diagnostics")] +impl From<&SourceFileRef> for miette::SourceSpan { + fn from(source: &SourceFileRef) -> Self { + source.span.into() + } +} + +/// Used to implement [miette::SpanContents] for [SourceFile] and [SourceFileRef] +#[cfg(feature = "diagnostics")] +struct ScopedSourceFileRef<'a> { + file: &'a SourceFile, + span: miette::SourceSpan, +} + +#[cfg(feature = "diagnostics")] +impl<'a> miette::SpanContents<'a> for ScopedSourceFileRef<'a> { + #[inline] + fn data(&self) -> &'a [u8] { + let start = self.span.offset(); + let end = start + self.span.len(); + &self.file.as_bytes()[start..end] + } + + #[inline] + fn span(&self) -> &miette::SourceSpan { + &self.span + } + + fn line(&self) -> usize { + let offset = self.span.offset() as u32; + self.file.content.line_index(offset.into()).to_usize() + } + + fn column(&self) -> usize { + let start = self.span.offset() as u32; + let end = start + self.span.len() as u32; + let span = SourceSpan::new(self.file.id(), start..end); + let loc = self.file.location(span); + loc.column.saturating_sub(1) as usize + } + + #[inline] + fn line_count(&self) -> usize { + self.file.line_count() + } + + #[inline] + fn name(&self) -> Option<&str> { + Some(self.file.content.path.as_ref()) + } + + #[inline] + fn language(&self) -> Option<&str> { + None + } +} + +#[cfg(feature = "diagnostics")] +impl miette::SourceCode for SourceFileRef { + #[inline] + fn read_span<'a>( + &'a self, + span: &miette::SourceSpan, + context_lines_before: usize, + context_lines_after: usize, + ) -> Result + 'a>, miette::MietteError> { + self.file.read_span(span, context_lines_before, context_lines_after) + } +} + +// SOURCE CONTENT +// ================================================================================================ + +/// Represents key information about a source file and its content: +/// +/// * The path to the file (or its name, in the case of virtual files) +/// * The content of the file +/// * The byte offsets of every line in the file, for use in looking up line/column information +#[derive(Clone)] +pub struct SourceContent { + /// The path (or name) of this file + path: Arc, + /// The underlying content of this file + content: Box, + /// The byte offsets for each line in this file + line_starts: Box<[ByteIndex]>, +} + +impl fmt::Debug for SourceContent { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("SourceContent") + .field("path", &self.path) + .field("size_in_bytes", &self.content.len()) + .field("line_count", &self.line_starts.len()) + .field("content", &self.content) + .finish() + } +} + +impl Eq for SourceContent {} + +impl PartialEq for SourceContent { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.path == other.path && self.content == other.content + } +} + +impl Ord for SourceContent { + #[inline] + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.path.cmp(&other.path).then_with(|| self.content.cmp(&other.content)) + } +} + +impl PartialOrd for SourceContent { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl core::hash::Hash for SourceContent { + fn hash(&self, state: &mut H) { + self.path.hash(state); + self.content.hash(state); + } +} + +impl SourceContent { + /// Create a new [SourceContent] from the (possibly virtual) file path, and its content as a + /// UTF-8 string. + /// + /// When created, the line starts for this file will be computed, which requires scanning the + /// file content once. + pub fn new(path: Arc, content: Box) -> Self { + let bytes = content.as_bytes(); + + assert!( + bytes.len() < u32::MAX as usize, + "unsupported source file: current maximum supported length in bytes is 2^32" + ); + + let line_starts = core::iter::once(ByteIndex(0)) + .chain(memchr::memchr_iter(b'\n', content.as_bytes()).filter_map(|mut offset| { + // Determine if the newline has any preceding escapes + let mut preceding_escapes = 0; + let line_start = offset + 1; + while let Some(prev_offset) = offset.checked_sub(1) { + if bytes[prev_offset] == b'\\' { + offset = prev_offset; + preceding_escapes += 1; + continue; + } + break; + } + + // If the newline is escaped, do not count it as a new line + let is_escaped = preceding_escapes > 0 && preceding_escapes % 2 != 0; + if is_escaped { + None + } else { + Some(ByteIndex(line_start as u32)) + } + })) + .collect::>() + .into_boxed_slice(); + + Self { + path, + content, + line_starts, + } + } + + /// Get the name of this source file + pub fn name(&self) -> Arc { + self.path.clone() + } + + /// Get the name of this source file as a [std::path::Path] + #[cfg(feature = "std")] + #[inline] + pub fn path(&self) -> &std::path::Path { + std::path::Path::new(self.path.as_ref()) + } + + /// Returns the underlying content as a string slice + #[inline(always)] + pub fn as_str(&self) -> &str { + self.content.as_ref() + } + + /// Returns the underlying content as a byte slice + #[inline(always)] + pub fn as_bytes(&self) -> &[u8] { + self.content.as_bytes() + } + + /// Returns the size in bytes of the underlying content + #[inline(always)] + pub fn len(&self) -> usize { + self.content.len() + } + + /// Returns true if the underlying content is empty + #[inline(always)] + pub fn is_empty(&self) -> bool { + self.content.is_empty() + } + + /// Returns the range of valid byte indices for this file + #[inline] + pub fn source_range(&self) -> Range { + ByteIndex(0)..ByteIndex(self.content.len() as u32) + } + + /// Returns a subset of the underlying content as a string slice. + /// + /// The bounds of the given span are character indices, _not_ byte indices. + /// + /// Returns `None` if the given span is out of bounds + #[inline(always)] + pub fn source_slice(&self, span: impl Into>) -> Option<&str> { + self.as_str().get(span.into()) + } + + /// Returns the byte index at which the line corresponding to `line_index` starts + /// + /// Returns `None` if the given index is out of bounds + pub fn line_start(&self, line_index: LineIndex) -> Option { + self.line_starts.get(line_index.to_usize()).copied().map(ByteIndex::from) + } + + /// Returns the index of the last line in this file + #[inline] + pub fn last_line_index(&self) -> LineIndex { + LineIndex(self.line_starts.len() as u32) + } + + /// Get the range of byte indices covered by the given line + pub fn line_range(&self, line_index: LineIndex) -> Option> { + let line_start = self.line_start(line_index)?; + match self.line_start(line_index + 1) { + Some(line_end) => Some(line_start..line_end), + None => Some(line_start..ByteIndex(self.content.len() as u32)), + } + } + + /// Get the index of the line to which `byte_index` belongs + pub fn line_index(&self, byte_index: ByteIndex) -> LineIndex { + match self.line_starts.binary_search(&byte_index) { + Ok(line) => LineIndex(line as u32), + Err(next_line) => LineIndex(next_line as u32 - 1), + } + } + + /// Get the [ByteIndex] corresponding to the given line and column indices. + /// + /// Returns `None` if the line or column indices are out of bounds. + pub fn line_column_to_offset( + &self, + line_index: LineIndex, + column_index: ColumnIndex, + ) -> Option { + let column_index = column_index.to_usize(); + let line_span = self.line_range(line_index)?; + let line_src = self + .content + .get(line_span.start.to_usize()..line_span.end.to_usize()) + .expect("invalid line boundaries: invalid utf-8"); + if line_src.len() < column_index { + return None; + } + let (pre, _) = line_src.split_at(column_index); + let start = line_span.start; + Some(start + ByteOffset::from_str_len(pre)) + } + + /// Get a [FileLineCol] corresponding to the line/column in this file at which `byte_index` + /// occurs + pub fn location(&self, byte_index: ByteIndex) -> Option { + let line_index = self.line_index(byte_index); + let line_start_index = self.line_start(line_index)?; + let line_src = self.content.get(line_start_index.to_usize()..byte_index.to_usize())?; + let column_index = ColumnIndex::from(line_src.chars().count() as u32); + Some(FileLineCol { + path: self.path.clone(), + line: line_index.number().get(), + column: column_index.number().get(), + }) + } +} + +// SOURCE CONTENT INDICES +// ================================================================================================ + +/// An index representing the offset in bytes from the start of a source file +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ByteIndex(u32); +impl ByteIndex { + /// Create a [ByteIndex] from a raw `u32` index + pub const fn new(index: u32) -> Self { + Self(index) + } + + /// Get the raw index as a usize + #[inline(always)] + pub const fn to_usize(self) -> usize { + self.0 as usize + } + + /// Get the raw index as a u32 + #[inline(always)] + pub const fn to_u32(self) -> u32 { + self.0 + } +} +impl core::ops::Add for ByteIndex { + type Output = ByteIndex; + + fn add(self, rhs: ByteOffset) -> Self { + Self((self.0 as i64 + rhs.0) as u32) + } +} +impl core::ops::Add for ByteIndex { + type Output = ByteIndex; + + fn add(self, rhs: u32) -> Self { + Self(self.0 + rhs) + } +} +impl core::ops::AddAssign for ByteIndex { + fn add_assign(&mut self, rhs: ByteOffset) { + *self = *self + rhs; + } +} +impl core::ops::AddAssign for ByteIndex { + fn add_assign(&mut self, rhs: u32) { + self.0 += rhs; + } +} +impl core::ops::Sub for ByteIndex { + type Output = ByteIndex; + + fn sub(self, rhs: ByteOffset) -> Self { + Self((self.0 as i64 - rhs.0) as u32) + } +} +impl core::ops::Sub for ByteIndex { + type Output = ByteIndex; + + fn sub(self, rhs: u32) -> Self { + Self(self.0 - rhs) + } +} +impl core::ops::SubAssign for ByteIndex { + fn sub_assign(&mut self, rhs: ByteOffset) { + *self = *self - rhs; + } +} +impl core::ops::SubAssign for ByteIndex { + fn sub_assign(&mut self, rhs: u32) { + self.0 -= rhs; + } +} +impl From for ByteIndex { + fn from(index: u32) -> Self { + Self(index) + } +} +impl From for u32 { + fn from(index: ByteIndex) -> Self { + index.0 + } +} + +/// An offset in bytes relative to some [ByteIndex] +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ByteOffset(i64); +impl ByteOffset { + /// Compute the offset in bytes represented by the given `char` + pub fn from_char_len(c: char) -> ByteOffset { + Self(c.len_utf8() as i64) + } + + /// Compute the offset in bytes represented by the given `str` + pub fn from_str_len(s: &str) -> ByteOffset { + Self(s.len() as i64) + } +} +impl core::ops::Add for ByteOffset { + type Output = ByteOffset; + + fn add(self, rhs: Self) -> Self { + Self(self.0 + rhs.0) + } +} +impl core::ops::AddAssign for ByteOffset { + fn add_assign(&mut self, rhs: Self) { + self.0 += rhs.0; + } +} +impl core::ops::Sub for ByteOffset { + type Output = ByteOffset; + + fn sub(self, rhs: Self) -> Self { + Self(self.0 - rhs.0) + } +} +impl core::ops::SubAssign for ByteOffset { + fn sub_assign(&mut self, rhs: Self) { + self.0 -= rhs.0; + } +} + +/// A zero-indexed line number +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct LineIndex(u32); +impl LineIndex { + /// Get a one-indexed number for display + pub const fn number(self) -> NonZeroU32 { + unsafe { NonZeroU32::new_unchecked(self.0 + 1) } + } + + /// Get the raw index as a usize + #[inline(always)] + pub const fn to_usize(self) -> usize { + self.0 as usize + } + + /// Add `offset` to this index, returning `None` on overflow + pub fn checked_add(self, offset: u32) -> Option { + self.0.checked_add(offset).map(Self) + } + + /// Subtract `offset` from this index, returning `None` on underflow + pub fn checked_sub(self, offset: u32) -> Option { + self.0.checked_sub(offset).map(Self) + } + + /// Add `offset` to this index, saturating to `u32::MAX` on overflow + pub const fn saturating_add(self, offset: u32) -> Self { + Self(self.0.saturating_add(offset)) + } + + /// Subtract `offset` from this index, saturating to `0` on overflow + pub const fn saturating_sub(self, offset: u32) -> Self { + Self(self.0.saturating_sub(offset)) + } +} +impl From for LineIndex { + fn from(index: u32) -> Self { + Self(index) + } +} +impl core::ops::Add for LineIndex { + type Output = LineIndex; + + fn add(self, rhs: u32) -> Self { + Self(self.0 + rhs) + } +} + +/// A zero-indexed column number +#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ColumnIndex(u32); +impl ColumnIndex { + /// Get a one-indexed number for display + pub const fn number(self) -> NonZeroU32 { + unsafe { NonZeroU32::new_unchecked(self.0 + 1) } + } + + /// Get the raw index as a usize + #[inline(always)] + pub const fn to_usize(self) -> usize { + self.0 as usize + } +} +impl From for ColumnIndex { + fn from(index: u32) -> Self { + Self(index) + } +} diff --git a/core/src/debuginfo/source_manager.rs b/core/src/debuginfo/source_manager.rs new file mode 100644 index 0000000000..e88e96d3dd --- /dev/null +++ b/core/src/debuginfo/source_manager.rs @@ -0,0 +1,369 @@ +use alloc::{ + collections::BTreeMap, + string::{String, ToString}, + sync::Arc, + vec::Vec, +}; + +use super::*; + +// SOURCE ID +// ================================================================================================ + +/// A [SourceId] represents the index/identifier associated with a unique source file in a +/// [SourceManager] implementation. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SourceId(u32); + +impl Default for SourceId { + fn default() -> Self { + Self::UNKNOWN + } +} + +impl SourceId { + pub const UNKNOWN: Self = Self(u32::MAX); + + /// Create a new [SourceId] from a `u32` value, but assert if the value is reserved + pub fn new(id: u32) -> Self { + assert_ne!(id, u32::MAX, "u32::MAX is a reserved value for SourceId::default()/UNKNOWN"); + + Self(id) + } + + /// Create a new [SourceId] from a raw `u32` value + #[inline(always)] + pub const fn new_unchecked(id: u32) -> Self { + Self(id) + } + + #[inline(always)] + pub const fn to_usize(self) -> usize { + self.0 as usize + } + + #[inline(always)] + pub const fn to_u32(self) -> u32 { + self.0 + } + + pub const fn is_unknown(&self) -> bool { + self.0 == u32::MAX + } +} + +impl TryFrom for SourceId { + type Error = (); + + #[inline] + fn try_from(id: usize) -> Result { + match u32::try_from(id) { + Ok(n) if n < u32::MAX => Ok(Self(n)), + _ => Err(()), + } + } +} + +// SOURCE MANAGER +// ================================================================================================ + +/// The set of errors which may be raised by a [SourceManager] +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SourceManagerError { + /// A [SourceId] was provided to a [SourceManager] which was allocated by a different + /// [SourceManager] + #[error("attempted to use an invalid source id")] + InvalidSourceId, + /// An attempt was made to read content using invalid byte indices + #[error("attempted to read content out of bounds")] + InvalidBounds, + /// An attempt to load a source file failed due to an I/O error + #[cfg(feature = "std")] + #[error(transparent)] + LoadFailed(#[from] std::io::Error), +} + +pub trait SourceManager { + /// Returns true if `file` is managed by this source manager + fn is_manager_of(&self, file: &SourceFile) -> bool { + match self.get(file.id()) { + Ok(found) => core::ptr::addr_eq(Arc::as_ptr(&found), file), + Err(_) => false, + } + } + /// Copies `file` into this source manager (if not already managed by this manager). + /// + /// The returned source file is guaranteed to be owned by this manager. + fn copy_into(&self, file: &SourceFile) -> Arc { + if let Ok(found) = self.get(file.id()) { + if core::ptr::addr_eq(Arc::as_ptr(&found), file) { + return found; + } + } + self.load_from_raw_parts(file.name(), file.content().clone()) + } + /// Load the given `content` into this [SourceManager] with `name` + fn load(&self, name: &str, content: String) -> Arc { + let name = Arc::from(name.to_string().into_boxed_str()); + let content = SourceContent::new(Arc::clone(&name), content.into_boxed_str()); + self.load_from_raw_parts(name, content) + } + /// Load content into this [SourceManager] from raw [SourceFile] components + fn load_from_raw_parts(&self, name: Arc, content: SourceContent) -> Arc; + /// Get the [SourceFile] corresponding to `id` + fn get(&self, id: SourceId) -> Result, SourceManagerError>; + /// Get the most recent [SourceFile] whose path is `path` + fn get_by_path(&self, path: &str) -> Option> { + self.find(path).and_then(|id| self.get(id).ok()) + } + /// Search for a source file named `name`, and return its [SourceId] if found. + fn find(&self, name: &str) -> Option; + /// Convert a [FileLineCol] to an equivalent [SourceSpan], if the referenced file is available + fn file_line_col_to_span(&self, loc: FileLineCol) -> Option; + /// Convert a [SourceSpan] to an equivalent [FileLineCol], if the span is valid + fn file_line_col(&self, span: SourceSpan) -> Result; + /// Convert a [Location] to an equivalent [SourceSpan], if the referenced file is available + fn location_to_span(&self, loc: Location) -> Option; + /// Convert a [SourceSpan] to an equivalent [Location], if the span is valid + fn location(&self, span: SourceSpan) -> Result; + /// Get the source associated with `id` as a string slice + fn source(&self, id: SourceId) -> Result<&str, SourceManagerError>; + /// Get the source corresponding to `span` as a string slice + fn source_slice(&self, span: SourceSpan) -> Result<&str, SourceManagerError>; +} + +impl SourceManager for Arc { + #[inline(always)] + fn is_manager_of(&self, file: &SourceFile) -> bool { + (**self).is_manager_of(file) + } + #[inline(always)] + fn copy_into(&self, file: &SourceFile) -> Arc { + (**self).copy_into(file) + } + #[inline(always)] + fn load(&self, name: &str, content: String) -> Arc { + (**self).load(name, content) + } + #[inline(always)] + fn load_from_raw_parts(&self, name: Arc, content: SourceContent) -> Arc { + (**self).load_from_raw_parts(name, content) + } + #[inline(always)] + fn get(&self, id: SourceId) -> Result, SourceManagerError> { + (**self).get(id) + } + #[inline(always)] + fn get_by_path(&self, path: &str) -> Option> { + (**self).get_by_path(path) + } + #[inline(always)] + fn find(&self, name: &str) -> Option { + (**self).find(name) + } + #[inline(always)] + fn file_line_col_to_span(&self, loc: FileLineCol) -> Option { + (**self).file_line_col_to_span(loc) + } + #[inline(always)] + fn file_line_col(&self, span: SourceSpan) -> Result { + (**self).file_line_col(span) + } + #[inline(always)] + fn location_to_span(&self, loc: Location) -> Option { + (**self).location_to_span(loc) + } + #[inline(always)] + fn location(&self, span: SourceSpan) -> Result { + (**self).location(span) + } + #[inline(always)] + fn source(&self, id: SourceId) -> Result<&str, SourceManagerError> { + (**self).source(id) + } + #[inline(always)] + fn source_slice(&self, span: SourceSpan) -> Result<&str, SourceManagerError> { + (**self).source_slice(span) + } +} + +#[cfg(feature = "std")] +pub trait SourceManagerExt: SourceManager { + /// Load the content of `path` into this [SourceManager], using the given path as the source + /// name. + fn load_file(&self, path: &std::path::Path) -> Result, SourceManagerError> { + let name = path.to_string_lossy(); + if let Some(existing) = self.get_by_path(name.as_ref()) { + return Ok(existing); + } + + let name = Arc::from(name.into_owned().into_boxed_str()); + let content = std::fs::read_to_string(path) + .map(|s| SourceContent::new(Arc::clone(&name), s.into_boxed_str())) + .map_err(SourceManagerError::LoadFailed)?; + + Ok(self.load_from_raw_parts(name, content)) + } +} + +#[cfg(feature = "std")] +impl SourceManagerExt for T {} + +// DEFAULT SOURCE MANAGER +// ================================================================================================ + +use crate::utils::sync::RwLock; + +#[derive(Default)] +pub struct DefaultSourceManager(RwLock); +impl Clone for DefaultSourceManager { + fn clone(&self) -> Self { + let manager = self.0.read(); + Self(RwLock::new(manager.clone())) + } +} + +#[derive(Default, Clone)] +struct DefaultSourceManagerImpl { + files: Vec>, + names: BTreeMap, SourceId>, +} + +impl DefaultSourceManagerImpl { + fn insert(&mut self, name: Arc, content: SourceContent) -> Arc { + // If we have previously inserted the same content with `name`, return the previously + // inserted source id + if let Some(file) = self.names.get(&name).copied().and_then(|id| { + let file = &self.files[id.to_usize()]; + if file.as_str() == content.as_str() { + Some(Arc::clone(file)) + } else { + None + } + }) { + return file; + } + let id = SourceId::try_from(self.files.len()) + .expect("system limit: source manager has exhausted its supply of source ids"); + let file = Arc::new(SourceFile::from_raw_parts(id, content)); + self.files.push(Arc::clone(&file)); + file + } + + fn get(&self, id: SourceId) -> Result, SourceManagerError> { + self.files + .get(id.to_usize()) + .cloned() + .ok_or(SourceManagerError::InvalidSourceId) + } + + fn get_by_path(&self, path: &str) -> Option> { + self.find(path).and_then(|id| self.get(id).ok()) + } + + fn find(&self, name: &str) -> Option { + self.names.get(name).copied() + } + + fn file_line_col_to_span(&self, loc: FileLineCol) -> Option { + let file = self + .names + .get(&loc.path) + .copied() + .and_then(|id| self.files.get(id.to_usize()))?; + file.line_column_to_span(loc.line, loc.column) + } + + fn file_line_col(&self, span: SourceSpan) -> Result { + self.files + .get(span.source_id().to_usize()) + .ok_or(SourceManagerError::InvalidSourceId) + .map(|file| file.location(span)) + } + + fn location_to_span(&self, loc: Location) -> Option { + let file = self + .names + .get(&loc.path) + .copied() + .and_then(|id| self.files.get(id.to_usize()))?; + + let max_len = ByteIndex::from(file.as_str().len() as u32); + if loc.start >= max_len || loc.end > max_len { + return None; + } + + Some(SourceSpan::new(file.id(), loc.start..loc.end)) + } + + fn location(&self, span: SourceSpan) -> Result { + self.files + .get(span.source_id().to_usize()) + .ok_or(SourceManagerError::InvalidSourceId) + .map(|file| Location::new(file.name(), span.start(), span.end())) + } +} + +impl SourceManager for DefaultSourceManager { + fn load_from_raw_parts(&self, name: Arc, content: SourceContent) -> Arc { + let mut manager = self.0.write(); + manager.insert(name, content) + } + + fn get(&self, id: SourceId) -> Result, SourceManagerError> { + let manager = self.0.read(); + manager.get(id) + } + + fn get_by_path(&self, path: &str) -> Option> { + let manager = self.0.read(); + manager.get_by_path(path) + } + + fn find(&self, name: &str) -> Option { + let manager = self.0.read(); + manager.find(name) + } + + fn file_line_col_to_span(&self, loc: FileLineCol) -> Option { + let manager = self.0.read(); + manager.file_line_col_to_span(loc) + } + + fn file_line_col(&self, span: SourceSpan) -> Result { + let manager = self.0.read(); + manager.file_line_col(span) + } + + fn location_to_span(&self, loc: Location) -> Option { + let manager = self.0.read(); + manager.location_to_span(loc) + } + + fn location(&self, span: SourceSpan) -> Result { + let manager = self.0.read(); + manager.location(span) + } + + fn source(&self, id: SourceId) -> Result<&str, SourceManagerError> { + let manager = self.0.read(); + let ptr = manager + .files + .get(id.to_usize()) + .ok_or(SourceManagerError::InvalidSourceId) + .map(|file| file.as_str() as *const str)?; + drop(manager); + // SAFETY: Because the lifetime of the returned reference is bound to the manager, and + // because we can only ever add files, not modify/remove them, this is safe. Exclusive + // access to the manager does _not_ mean exclusive access to the contents of previously + // added source files + Ok(unsafe { &*ptr }) + } + + fn source_slice(&self, span: SourceSpan) -> Result<&str, SourceManagerError> { + self.source(span.source_id())? + .get(span.into_slice_index()) + .ok_or(SourceManagerError::InvalidBounds) + } +} diff --git a/assembly/src/parser/span.rs b/core/src/debuginfo/span.rs similarity index 68% rename from assembly/src/parser/span.rs rename to core/src/debuginfo/span.rs index e6f663d299..592ffdeacc 100644 --- a/assembly/src/parser/span.rs +++ b/core/src/debuginfo/span.rs @@ -5,14 +5,34 @@ use core::{ ops::{Bound, Deref, DerefMut, Index, Range, RangeBounds}, }; -use crate::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable}; +use super::{ByteIndex, ByteOffset, SourceId}; +use crate::utils::{ByteReader, ByteWriter, Deserializable, DeserializationError, Serializable}; /// This trait should be implemented for any type that has an associated [SourceSpan]. pub trait Spanned { fn span(&self) -> SourceSpan; } -impl Spanned for &T { +impl Spanned for SourceSpan { + #[inline(always)] + fn span(&self) -> SourceSpan { + *self + } +} + +impl Spanned for alloc::boxed::Box { + fn span(&self) -> SourceSpan { + (**self).span() + } +} + +impl Spanned for alloc::rc::Rc { + fn span(&self) -> SourceSpan { + (**self).span() + } +} + +impl Spanned for alloc::sync::Arc { fn span(&self) -> SourceSpan { (**self).span() } @@ -57,9 +77,10 @@ impl Span { /// Creates a span for `spanned` representing a single location, `offset`. #[inline] - pub fn at(offset: usize, spanned: T) -> Self { + pub fn at(source_id: SourceId, offset: usize, spanned: T) -> Self { + let offset = u32::try_from(offset).expect("invalid source offset: too large"); Self { - span: SourceSpan::at(offset.try_into().expect("invalid source offset: too large")), + span: SourceSpan::at(source_id, offset), spanned, } } @@ -78,6 +99,12 @@ impl Span { self.span } + /// Gets a reference to the spanned item. + #[inline(always)] + pub const fn inner(&self) -> &T { + &self.spanned + } + /// Applies a transformation to the spanned value while retaining the same [SourceSpan]. #[inline] pub fn map(self, mut f: F) -> Span @@ -113,16 +140,14 @@ impl Span { /// Shifts the span right by `count` units #[inline] - pub fn shift(&mut self, count: usize) { - let count: u32 = count.try_into().expect("invalid count: must be smaller than 2^32"); + pub fn shift(&mut self, count: ByteOffset) { self.span.start += count; self.span.end += count; } /// Extends the end of the span by `count` units. #[inline] - pub fn extend(&mut self, count: usize) { - let count: u32 = count.try_into().expect("invalid count: must be smaller than 2^32"); + pub fn extend(&mut self, count: ByteOffset) { self.span.end += count; } @@ -229,23 +254,28 @@ impl Hash for Span { } } -/// Serialization impl Span { - pub fn write_into(&self, target: &mut W, options: crate::ast::AstSerdeOptions) { - if options.debug_info { + pub fn write_into_with_options(&self, target: &mut W, debug: bool) { + if debug { self.span.write_into(target); } self.spanned.write_into(target); } } -/// Deserialization +impl Serializable for Span { + fn write_into(&self, target: &mut W) { + self.span.write_into(target); + self.spanned.write_into(target); + } +} + impl Span { - pub fn read_from( + pub fn read_from_with_options( source: &mut R, - options: crate::ast::AstSerdeOptions, + debug: bool, ) -> Result { - let span = if options.debug_info { + let span = if debug { SourceSpan::read_from(source)? } else { SourceSpan::default() @@ -255,13 +285,6 @@ impl Span { } } -impl Serializable for Span { - fn write_into(&self, target: &mut W) { - self.span.write_into(target); - self.spanned.write_into(target); - } -} - impl Deserializable for Span { fn read_from(source: &mut R) -> Result { let span = SourceSpan::read_from(source)?; @@ -286,43 +309,89 @@ impl Deserializable for Span { /// to produce nice errors with it compared to this representation. #[derive(Debug, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct SourceSpan { - start: u32, - end: u32, + source_id: SourceId, + start: ByteIndex, + end: ByteIndex, } +#[derive(Debug, thiserror::Error)] +#[error("invalid byte index range: maximum supported byte index is 2^32")] +pub struct InvalidByteIndexRange; + impl SourceSpan { + /// A sentinel [SourceSpan] that indicates the span is unknown/invalid + pub const UNKNOWN: Self = Self { + source_id: SourceId::UNKNOWN, + start: ByteIndex::new(0), + end: ByteIndex::new(0), + }; + /// Creates a new [SourceSpan] from the given range. - pub fn new(range: Range) -> Self { + pub fn new(source_id: SourceId, range: Range) -> Self + where + B: Into, + { Self { - start: range.start, - end: range.end, + source_id, + start: range.start.into(), + end: range.end.into(), } } /// Creates a new [SourceSpan] for a specific offset. - pub fn at(offset: u32) -> Self { + pub fn at(source_id: SourceId, offset: impl Into) -> Self { + let offset = offset.into(); Self { + source_id, start: offset, end: offset, } } + /// Try to create a new [SourceSpan] from the given range with `usize` bounds. + pub fn try_from_range( + source_id: SourceId, + range: Range, + ) -> Result { + const MAX: usize = u32::MAX as usize; + if range.start > MAX || range.end > MAX { + return Err(InvalidByteIndexRange); + } + + Ok(SourceSpan { + source_id, + start: ByteIndex::from(range.start as u32), + end: ByteIndex::from(range.end as u32), + }) + } + + /// Returns `true` if this [SourceSpan] represents the unknown span + pub const fn is_unknown(&self) -> bool { + self.source_id.is_unknown() + } + + /// Get the [SourceId] associated with this source span + #[inline(always)] + pub fn source_id(&self) -> SourceId { + self.source_id + } + /// Gets the offset in bytes corresponding to the start of this span (inclusive). #[inline(always)] - pub fn start(&self) -> usize { - self.start as usize + pub fn start(&self) -> ByteIndex { + self.start } /// Gets the offset in bytes corresponding to the end of this span (exclusive). #[inline(always)] - pub fn end(&self) -> usize { - self.end as usize + pub fn end(&self) -> ByteIndex { + self.end } /// Gets the length of this span in bytes. #[inline(always)] pub fn len(&self) -> usize { - (self.end - self.start) as usize + self.end.to_usize() - self.start.to_usize() } /// Returns true if this span is empty. @@ -333,44 +402,41 @@ impl SourceSpan { /// Converts this span into a [`Range`]. #[inline] pub fn into_range(self) -> Range { - self.start..self.end + self.start.to_u32()..self.end.to_u32() } -} -impl Serializable for SourceSpan { - fn write_into(&self, target: &mut W) { - target.write_u32(self.start); - target.write_u32(self.end) + /// Converts this span into a [`Range`]. + #[inline] + pub fn into_slice_index(self) -> Range { + self.start.to_usize()..self.end.to_usize() } } -impl Deserializable for SourceSpan { - fn read_from(source: &mut R) -> Result { - let start = source.read_u32()?; - let end = source.read_u32()?; - Ok(Self { start, end }) +#[cfg(feature = "diagnostics")] +impl From for miette::SourceSpan { + fn from(span: SourceSpan) -> Self { + Self::new(miette::SourceOffset::from(span.start().to_usize()), span.len()) } } -impl TryFrom> for SourceSpan { - type Error = (); - - fn try_from(range: Range) -> Result { - const MAX: usize = u32::MAX as usize; - if range.start > MAX || range.end > MAX { - return Err(()); - } - Ok(SourceSpan { - start: range.start as u32, - end: range.end as u32, - }) +impl Serializable for SourceSpan { + fn write_into(&self, target: &mut W) { + target.write_u32(self.source_id.to_u32()); + target.write_u32(self.start.into()); + target.write_u32(self.end.into()) } } -impl From> for SourceSpan { - #[inline(always)] - fn from(range: Range) -> Self { - Self::new(range) +impl Deserializable for SourceSpan { + fn read_from(source: &mut R) -> Result { + let source_id = SourceId::new_unchecked(source.read_u32()?); + let start = ByteIndex::from(source.read_u32()?); + let end = ByteIndex::from(source.read_u32()?); + Ok(Self { + source_id, + start, + end, + }) } } @@ -381,10 +447,10 @@ impl From for Range { } } -impl From for miette::SourceSpan { - #[inline] +impl From for Range { + #[inline(always)] fn from(span: SourceSpan) -> Self { - miette::SourceSpan::new(miette::SourceOffset::from(span.start as usize), span.len()) + span.into_slice_index() } } @@ -393,18 +459,18 @@ impl Index for [u8] { #[inline] fn index(&self, index: SourceSpan) -> &Self::Output { - &self[index.start()..index.end()] + &self[index.start().to_usize()..index.end().to_usize()] } } -impl RangeBounds for SourceSpan { +impl RangeBounds for SourceSpan { #[inline(always)] - fn start_bound(&self) -> Bound<&u32> { + fn start_bound(&self) -> Bound<&ByteIndex> { Bound::Included(&self.start) } #[inline(always)] - fn end_bound(&self) -> Bound<&u32> { + fn end_bound(&self) -> Bound<&ByteIndex> { Bound::Excluded(&self.end) } } diff --git a/core/src/lib.rs b/core/src/lib.rs index 7be244471b..87e618b366 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -50,6 +50,7 @@ assertion failed: `(left matches right)` } pub mod chiplets; +pub mod debuginfo; pub mod errors; mod program; diff --git a/core/src/mast/serialization/basic_block_data_builder.rs b/core/src/mast/serialization/basic_block_data_builder.rs index 78a045d6c4..9aa95ee9fe 100644 --- a/core/src/mast/serialization/basic_block_data_builder.rs +++ b/core/src/mast/serialization/basic_block_data_builder.rs @@ -102,6 +102,17 @@ impl BasicBlockDataBuilder { self.data.push(assembly_op.num_cycles()); self.data.write_bool(assembly_op.should_break()); + // source location + let loc = assembly_op.location(); + self.data.write_bool(loc.is_some()); + if let Some(loc) = loc { + let str_index_in_table = + self.string_table_builder.add_string(loc.path.as_ref()); + self.data.write_usize(str_index_in_table); + self.data.write_u32(loc.start.to_u32()); + self.data.write_u32(loc.end.to_u32()); + } + // context name { let str_index_in_table = diff --git a/core/src/mast/serialization/basic_block_data_decoder.rs b/core/src/mast/serialization/basic_block_data_decoder.rs index 78dd77215f..f5a38a63dc 100644 --- a/core/src/mast/serialization/basic_block_data_decoder.rs +++ b/core/src/mast/serialization/basic_block_data_decoder.rs @@ -3,19 +3,38 @@ use crate::{ }; use super::{decorator::EncodedDecoratorVariant, DataOffset, StringIndex}; -use alloc::{string::String, vec::Vec}; +use alloc::{string::String, sync::Arc, vec::Vec}; +use core::cell::RefCell; use miden_crypto::Felt; use winter_utils::{ByteReader, Deserializable, DeserializationError, SliceReader}; pub struct BasicBlockDataDecoder<'a> { data: &'a [u8], strings: &'a [DataOffset], + /// This field is used to allocate an `Arc` for any string in `strings` where the decoder + /// requests a reference-counted string rather than a fresh allocation as a `String`. + /// + /// Currently, this is only used for debug information (source file names), but most cases + /// where strings are stored in MAST are stored as `Arc` in practice, we just haven't yet + /// updated all of the decoders. + /// + /// We lazily allocate an `Arc` when strings are decoded as an `Arc`, but the underlying + /// string data corresponds to the same index in `strings`. All future requests for a + /// ref-counted string we've allocated an `Arc` for, will clone the `Arc` rather than + /// allocate a fresh string. + refc_strings: Vec>>>, } /// Constructors impl<'a> BasicBlockDataDecoder<'a> { pub fn new(data: &'a [u8], strings: &'a [DataOffset]) -> Self { - Self { data, strings } + let mut refc_strings = Vec::with_capacity(strings.len()); + refc_strings.resize(strings.len(), RefCell::new(None)); + Self { + data, + strings, + refc_strings, + } } } @@ -139,6 +158,21 @@ impl<'a> BasicBlockDataDecoder<'a> { let num_cycles = data_reader.read_u8()?; let should_break = data_reader.read_bool()?; + // source location + let location = if data_reader.read_bool()? { + let str_index_in_table = data_reader.read_usize()?; + let path = self.read_arc_str(str_index_in_table)?; + let start = data_reader.read_u32()?; + let end = data_reader.read_u32()?; + Some(crate::debuginfo::Location { + path, + start: start.into(), + end: end.into(), + }) + } else { + None + }; + let context_name = { let str_index_in_table = data_reader.read_usize()?; self.read_string(str_index_in_table)? @@ -149,7 +183,13 @@ impl<'a> BasicBlockDataDecoder<'a> { self.read_string(str_index_in_table)? }; - Ok(Decorator::AsmOp(AssemblyOp::new(context_name, num_cycles, op, should_break))) + Ok(Decorator::AsmOp(AssemblyOp::new( + location, + context_name, + num_cycles, + op, + should_break, + ))) } EncodedDecoratorVariant::DebugOptionsStackAll => { Ok(Decorator::Debug(DebugOptions::StackAll)) @@ -188,6 +228,17 @@ impl<'a> BasicBlockDataDecoder<'a> { } } + fn read_arc_str(&self, str_idx: StringIndex) -> Result, DeserializationError> { + if let Some(cached) = self.refc_strings.get(str_idx).and_then(|cell| cell.borrow().clone()) + { + return Ok(cached); + } + + let string = Arc::from(self.read_string(str_idx)?.into_boxed_str()); + *self.refc_strings[str_idx].borrow_mut() = Some(Arc::clone(&string)); + Ok(string) + } + fn read_string(&self, str_idx: StringIndex) -> Result { let str_offset = self.strings.get(str_idx).copied().ok_or_else(|| { DeserializationError::InvalidValue(format!("invalid index in strings table: {str_idx}")) diff --git a/core/src/mast/serialization/tests.rs b/core/src/mast/serialization/tests.rs index 6b0ad82ca5..3cfc41480b 100644 --- a/core/src/mast/serialization/tests.rs +++ b/core/src/mast/serialization/tests.rs @@ -1,4 +1,4 @@ -use alloc::string::ToString; +use alloc::{string::ToString, sync::Arc}; use miden_crypto::{hash::rpo::RpoDigest, Felt}; use super::*; @@ -281,6 +281,11 @@ fn serialize_deserialize_all_nodes() { ( 15, Decorator::AsmOp(AssemblyOp::new( + Some(crate::debuginfo::Location { + path: Arc::from("test"), + start: 42.into(), + end: 43.into(), + }), "context".to_string(), 15, "op".to_string(), diff --git a/core/src/operations/decorators/assembly_op.rs b/core/src/operations/decorators/assembly_op.rs index 56689724cf..4676e76e1b 100644 --- a/core/src/operations/decorators/assembly_op.rs +++ b/core/src/operations/decorators/assembly_op.rs @@ -1,30 +1,45 @@ use alloc::string::String; use core::fmt; +use crate::debuginfo::Location; + // ASSEMBLY OP // ================================================================================================ /// Contains information corresponding to an assembly instruction (only applicable in debug mode). #[derive(Clone, Debug, Eq, PartialEq)] pub struct AssemblyOp { + location: Option, context_name: String, - num_cycles: u8, op: String, + num_cycles: u8, should_break: bool, } impl AssemblyOp { /// Returns [AssemblyOp] instantiated with the specified assembly instruction string and number /// of cycles it takes to execute the assembly instruction. - pub fn new(context_name: String, num_cycles: u8, op: String, should_break: bool) -> Self { + pub fn new( + location: Option, + context_name: String, + num_cycles: u8, + op: String, + should_break: bool, + ) -> Self { Self { + location, context_name, - num_cycles, op, + num_cycles, should_break, } } + /// Returns the [Location] for this operation, if known + pub fn location(&self) -> Option<&Location> { + self.location.as_ref() + } + /// Returns the context name for this operation. pub fn context_name(&self) -> &str { &self.context_name @@ -52,6 +67,11 @@ impl AssemblyOp { pub fn set_num_cycles(&mut self, num_cycles: u8) { self.num_cycles = num_cycles; } + + /// Change the [Location] of this [AssemblyOp] + pub fn set_location(&mut self, location: Location) { + self.location = Some(location); + } } impl fmt::Display for AssemblyOp { diff --git a/core/src/utils/mod.rs b/core/src/utils/mod.rs index 823cd48612..af612567a3 100644 --- a/core/src/utils/mod.rs +++ b/core/src/utils/mod.rs @@ -1,3 +1,5 @@ +pub mod sync; + use crate::Felt; use alloc::vec::Vec; use core::{ diff --git a/core/src/utils/sync.rs b/core/src/utils/sync.rs new file mode 100644 index 0000000000..0c96ff9586 --- /dev/null +++ b/core/src/utils/sync.rs @@ -0,0 +1,312 @@ +#[cfg(feature = "std")] +pub use parking_lot::{RwLock, RwLockReadGuard, RwLockWriteGuard}; + +#[cfg(not(feature = "std"))] +pub use self::rwlock::{RwLock, RwLockReadGuard, RwLockWriteGuard}; + +pub mod rwlock { + #[cfg(not(loom))] + use core::{ + hint, + sync::atomic::{AtomicUsize, Ordering}, + }; + + #[cfg(loom)] + use loom::{ + hint, + sync::atomic::{AtomicUsize, Ordering}, + }; + + use lock_api::RawRwLock; + + /// An implementation of a reader-writer lock, based on a spinlock primitive, no-std compatible + /// + /// See [lock_api::RwLock] for usage. + pub type RwLock = lock_api::RwLock; + + /// See [lock_api::RwLockReadGuard] for usage. + pub type RwLockReadGuard<'a, T> = lock_api::RwLockReadGuard<'a, Spinlock, T>; + + /// See [lock_api::RwLockWriteGuard] for usage. + pub type RwLockWriteGuard<'a, T> = lock_api::RwLockWriteGuard<'a, Spinlock, T>; + + /// The underlying raw reader-writer primitive that implements [lock_api::RawRwLock] + /// + /// This is fundamentally a spinlock, in that blocking operations on the lock will spin until + /// they succeed in acquiring/releasing the lock. + /// + /// To acheive the ability to share the underlying data with multiple readers, or hold + /// exclusive access for one writer, the lock state is based on a "locked" count, where shared + /// access increments the count by an even number, and acquiring exclusive access relies on the + /// use of the lowest order bit to stop further shared acquisition, and indicate that the lock + /// is exclusively held (the difference between the two is irrelevant from the perspective of + /// a thread attempting to acquire the lock, but internally the state uses `usize::MAX` as the + /// "exlusively locked" sentinel). + /// + /// This mechanism gets us the following: + /// + /// * Whether the lock has been acquired (shared or exclusive) + /// * Whether the lock is being exclusively acquired + /// * How many times the lock has been acquired + /// * Whether the acquisition(s) are exclusive or shared + /// + /// Further implementation details, such as how we manage draining readers once an attempt to + /// exclusively acquire the lock occurs, are described below. + /// + /// NOTE: This is a simple implementation, meant for use in no-std environments; there are much + /// more robust/performant implementations available when OS primitives can be used. + pub struct Spinlock { + /// The state of the lock, primarily representing the acquisition count, but relying on + /// the distinction between even and odd values to indicate whether or not exclusive access + /// is being acquired. + state: AtomicUsize, + /// A counter used to wake a parked writer once the last shared lock is released during + /// acquisition of an exclusive lock. The actual count is not acutally important, and + /// simply wraps around on overflow, but what is important is that when the value changes, + /// the writer will wake and resume attempting to acquire the exclusive lock. + writer_wake_counter: AtomicUsize, + } + + impl Default for Spinlock { + #[inline(always)] + fn default() -> Self { + Self::new() + } + } + + impl Spinlock { + #[cfg(not(loom))] + pub const fn new() -> Self { + Self { + state: AtomicUsize::new(0), + writer_wake_counter: AtomicUsize::new(0), + } + } + + #[cfg(loom)] + pub fn new() -> Self { + Self { + state: AtomicUsize::new(0), + writer_wake_counter: AtomicUsize::new(0), + } + } + } + + unsafe impl RawRwLock for Spinlock { + #[cfg(loom)] + const INIT: Spinlock = unimplemented!(); + + #[cfg(not(loom))] + // This is intentional on the part of the [RawRwLock] API, basically a hack to provide + // initial values as static items. + #[allow(clippy::declare_interior_mutable_const)] + const INIT: Spinlock = Spinlock::new(); + + type GuardMarker = lock_api::GuardSend; + + /// The operation invoked when calling `RwLock::read`, blocks the caller until acquired + fn lock_shared(&self) { + let mut s = self.state.load(Ordering::Relaxed); + loop { + // If the exclusive bit is unset, attempt to acquire a read lock + if s & 1 == 0 { + match self.state.compare_exchange_weak( + s, + s + 2, + Ordering::Acquire, + Ordering::Relaxed, + ) { + Ok(_) => return, + // Someone else beat us to the punch and acquired a lock + Err(e) => s = e, + } + } + // If an exclusive lock is held/being acquired, loop until the lock state changes + // at which point, try to acquire the lock again + if s & 1 == 1 { + loop { + let next = self.state.load(Ordering::Relaxed); + if s == next { + hint::spin_loop(); + continue; + } else { + s = next; + break; + } + } + } + } + } + + /// The operation invoked when calling `RwLock::try_read`, returns whether or not the + /// lock was acquired + fn try_lock_shared(&self) -> bool { + let s = self.state.load(Ordering::Relaxed); + if s & 1 == 0 { + self.state + .compare_exchange_weak(s, s + 2, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } else { + false + } + } + + /// The operation invoked when dropping a `RwLockReadGuard` + unsafe fn unlock_shared(&self) { + if self.state.fetch_sub(2, Ordering::Release) == 3 { + // The lock is being exclusively acquired, and we're the last shared acquisition + // to be released, so wake the writer by incrementing the wake counter + self.writer_wake_counter.fetch_add(1, Ordering::Release); + } + } + + /// The operation invoked when calling `RwLock::write`, blocks the caller until acquired + fn lock_exclusive(&self) { + let mut s = self.state.load(Ordering::Relaxed); + loop { + // Attempt to acquire the lock immediately, or complete acquistion of the lock + // if we're continuing the loop after acquiring the exclusive bit. If another + // thread acquired it first, we race to be the first thread to acquire it once + // released, by busy looping here. + if s <= 1 { + match self.state.compare_exchange( + s, + usize::MAX, + Ordering::Acquire, + Ordering::Relaxed, + ) { + Ok(_) => return, + Err(e) => { + s = e; + hint::spin_loop(); + continue; + } + } + } + + // Only shared locks have been acquired, attempt to acquire the exclusive bit, + // which will prevent further shared locks from being acquired. It does not + // in and of itself grant us exclusive access however. + if s & 1 == 0 { + if let Err(e) = + self.state.compare_exchange(s, s + 1, Ordering::Relaxed, Ordering::Relaxed) + { + // The lock state has changed before we could acquire the exclusive bit, + // update our view of the lock state and try again + s = e; + continue; + } + } + + // We've acquired the exclusive bit, now we need to busy wait until all shared + // acquisitions are released. + let w = self.writer_wake_counter.load(Ordering::Acquire); + s = self.state.load(Ordering::Relaxed); + + // "Park" the thread here (by busy looping), until the release of the last shared + // lock, which is communicated to us by it incrementing the wake counter. + if s >= 2 { + while self.writer_wake_counter.load(Ordering::Acquire) == w { + hint::spin_loop(); + } + s = self.state.load(Ordering::Relaxed); + } + + // All shared locks have been released, go back to the top and try to complete + // acquisition of exclusive access. + } + } + + /// The operation invoked when calling `RwLock::try_write`, returns whether or not the + /// lock was acquired + fn try_lock_exclusive(&self) -> bool { + let s = self.state.load(Ordering::Relaxed); + if s <= 1 { + self.state + .compare_exchange(s, usize::MAX, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + } else { + false + } + } + + /// The operation invoked when dropping a `RwLockWriteGuard` + unsafe fn unlock_exclusive(&self) { + // Infallible, as we hold an exclusive lock + // + // Note the use of `Release` ordering here, which ensures any loads of the lock state + // by other threads, are ordered after this store. + self.state.store(0, Ordering::Release); + // This fetch_add isn't important for signaling purposes, however it serves a key + // purpose, in that it imposes a memory ordering on any loads of this field that + // have an `Acquire` ordering, i.e. they will read the value stored here. Without + // a `Release` store, loads/stores of this field could be reordered relative to + // each other. + self.writer_wake_counter.fetch_add(1, Ordering::Release); + } + } +} + +#[cfg(all(loom, test))] +mod test { + use super::rwlock::{RwLock, Spinlock}; + use alloc::vec::Vec; + use loom::{model::Builder, sync::Arc}; + + #[test] + fn test_rwlock_loom() { + let mut builder = Builder::default(); + builder.max_duration = Some(std::time::Duration::from_secs(60)); + builder.log = true; + builder.check(|| { + let raw_rwlock = Spinlock::new(); + let n = Arc::new(RwLock::from_raw(raw_rwlock, 0usize)); + let mut readers = Vec::new(); + let mut writers = Vec::new(); + + let num_readers = 2; + let num_writers = 2; + let num_iterations = 2; + + // Readers should never observe a non-zero value + for _ in 0..num_readers { + let n0 = n.clone(); + let t = loom::thread::spawn(move || { + for _ in 0..num_iterations { + let guard = n0.read(); + assert_eq!(*guard, 0); + } + }); + + readers.push(t); + } + + // Writers should never observe a non-zero value once they've + // acquired the lock, and should never observe a value > 1 + // while holding the lock + for _ in 0..num_writers { + let n0 = n.clone(); + let t = loom::thread::spawn(move || { + for _ in 0..num_iterations { + let mut guard = n0.write(); + assert_eq!(*guard, 0); + *guard += 1; + assert_eq!(*guard, 1); + *guard -= 1; + assert_eq!(*guard, 0); + } + }); + + writers.push(t); + } + + for t in readers { + t.join().unwrap(); + } + + for t in writers { + t.join().unwrap(); + } + }) + } +} diff --git a/miden/benches/program_compilation.rs b/miden/benches/program_compilation.rs index 417e506918..b2065edc8a 100644 --- a/miden/benches/program_compilation.rs +++ b/miden/benches/program_compilation.rs @@ -7,6 +7,7 @@ fn program_compilation(c: &mut Criterion) { let mut group = c.benchmark_group("program_compilation"); group.measurement_time(Duration::from_secs(10)); + let stdlib = StdLibrary::default(); group.bench_function("sha256", |bench| { let source = " use.std::crypto::hashes::sha256 @@ -16,9 +17,7 @@ fn program_compilation(c: &mut Criterion) { end"; bench.iter(|| { let mut assembler = Assembler::default(); - assembler - .add_compiled_library(StdLibrary::default()) - .expect("failed to load stdlib"); + assembler.add_compiled_library(&stdlib).expect("failed to load stdlib"); assembler.assemble_program(source).expect("Failed to compile test source.") }); diff --git a/miden/benches/program_execution.rs b/miden/benches/program_execution.rs index 6b7556f15e..4aa21ff8ce 100644 --- a/miden/benches/program_execution.rs +++ b/miden/benches/program_execution.rs @@ -8,6 +8,10 @@ fn program_execution(c: &mut Criterion) { let mut group = c.benchmark_group("program_execution"); group.measurement_time(Duration::from_secs(10)); + let stdlib = StdLibrary::default(); + let mut host = DefaultHost::default(); + host.load_mast_forest(stdlib.as_ref().mast_forest().clone()); + group.bench_function("sha256", |bench| { let source = " use.std::crypto::hashes::sha256 @@ -16,18 +20,11 @@ fn program_execution(c: &mut Criterion) { exec.sha256::hash_2to1 end"; let mut assembler = Assembler::default(); - assembler - .add_compiled_library(StdLibrary::default()) - .expect("failed to load stdlib"); + assembler.add_compiled_library(&stdlib).expect("failed to load stdlib"); let program: Program = assembler.assemble_program(source).expect("Failed to compile test source."); bench.iter(|| { - execute( - &program, - StackInputs::default(), - DefaultHost::default(), - ExecutionOptions::default(), - ) + execute(&program, StackInputs::default(), host.clone(), ExecutionOptions::default()) }); }); diff --git a/miden/src/cli/bundle.rs b/miden/src/cli/bundle.rs index d660982d55..6dd8f3b4f5 100644 --- a/miden/src/cli/bundle.rs +++ b/miden/src/cli/bundle.rs @@ -5,7 +5,7 @@ use assembly::{ LibraryNamespace, Version, }; use clap::Parser; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; #[derive(Debug, Clone, Parser)] #[clap( @@ -40,11 +40,12 @@ impl BundleCmd { .into_owned(), }; + let source_manager = Arc::new(assembly::DefaultSourceManager::default()); let library_namespace = namespace.parse::().expect("invalid base namespace"); // TODO: Add version to `Library` let _version = self.version.parse::().expect("invalid cargo version"); - let stdlib = CompiledLibrary::from_dir(&self.dir, library_namespace)?; + let stdlib = CompiledLibrary::from_dir(&self.dir, library_namespace, source_manager)?; // write the masl output let options = AstSerdeOptions::new(false, false); diff --git a/miden/src/cli/data.rs b/miden/src/cli/data.rs index 333f187236..77e3eab4c7 100644 --- a/miden/src/cli/data.rs +++ b/miden/src/cli/data.rs @@ -17,6 +17,7 @@ use std::{ fs, io::Write, path::{Path, PathBuf}, + sync::Arc, }; use stdlib::StdLibrary; pub use tracing::{event, instrument, Level}; @@ -386,20 +387,35 @@ impl OutputFile { pub struct ProgramFile { ast: Box, path: PathBuf, + source_manager: Arc, } /// Helper methods to interact with masm program file. impl ProgramFile { /// Reads the masm file at the specified path and parses it into a [ProgramFile]. - #[instrument(name = "read_program_file", fields(path = %path.display()))] - pub fn read(path: &PathBuf) -> Result { + pub fn read(path: impl AsRef) -> Result { + let source_manager = Arc::new(assembly::DefaultSourceManager::default()); + Self::read_with(path, source_manager) + } + + /// Reads the masm file at the specified path and parses it into a [ProgramFile], using the + /// provided [assembly::SourceManager] implementation. + #[instrument(name = "read_program_file", skip(source_manager), fields(path = %path.as_ref().display()))] + pub fn read_with( + path: impl AsRef, + source_manager: Arc, + ) -> Result { // parse the program into an AST - let ast = Module::parse_file(LibraryNamespace::Exec.into(), ModuleKind::Executable, path) + let path = path.as_ref(); + let mut parser = Module::parser(ModuleKind::Executable); + let ast = parser + .parse_file(LibraryNamespace::Exec.into(), path, &source_manager) .wrap_err_with(|| format!("Failed to parse program file `{}`", path.display()))?; Ok(Self { ast, - path: path.clone(), + path: path.to_path_buf(), + source_manager, }) } @@ -410,7 +426,8 @@ impl ProgramFile { I: IntoIterator, { // compile program - let mut assembler = Assembler::default().with_debug_mode(debug.is_on()); + let mut assembler = + Assembler::new(self.source_manager.clone()).with_debug_mode(debug.is_on()); assembler .add_compiled_library(StdLibrary::default()) .wrap_err("Failed to load stdlib")?; diff --git a/miden/src/cli/debug/executor.rs b/miden/src/cli/debug/executor.rs index 56ff471950..29e26a42ca 100644 --- a/miden/src/cli/debug/executor.rs +++ b/miden/src/cli/debug/executor.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use super::DebugCommand; use miden_vm::{ math::Felt, DefaultHost, MemAdviceProvider, Program, StackInputs, VmState, VmStateIterator, @@ -7,6 +9,9 @@ use miden_vm::{ pub struct DebugExecutor { vm_state_iter: VmStateIterator, vm_state: VmState, + // TODO(pauls): Use this to render source-level diagnostics when program errors are encountered + #[allow(unused)] + source_manager: Arc, } impl DebugExecutor { @@ -20,6 +25,7 @@ impl DebugExecutor { program: Program, stack_inputs: StackInputs, advice_provider: MemAdviceProvider, + source_manager: Arc, ) -> Result { let mut vm_state_iter = processor::execute_iter(&program, stack_inputs, DefaultHost::new(advice_provider)); @@ -34,6 +40,7 @@ impl DebugExecutor { Ok(Self { vm_state_iter, vm_state, + source_manager, }) } diff --git a/miden/src/cli/debug/mod.rs b/miden/src/cli/debug/mod.rs index 3ba4ea4a2d..19692058e8 100644 --- a/miden/src/cli/debug/mod.rs +++ b/miden/src/cli/debug/mod.rs @@ -2,7 +2,7 @@ use super::data::{Debug, InputFile, Libraries, ProgramFile}; use assembly::diagnostics::Report; use clap::Parser; use rustyline::{error::ReadlineError, Config, DefaultEditor, EditMode}; -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; mod command; use command::DebugCommand; @@ -33,12 +33,14 @@ impl DebugCmd { println!("Debug program"); println!("============================================================"); + let source_manager = Arc::new(assembly::DefaultSourceManager::default()); + // load libraries from files let libraries = Libraries::new(&self.library_paths)?; // load program from file and compile - let program = - ProgramFile::read(&self.assembly_file)?.compile(&Debug::On, &libraries.libraries)?; + let program = ProgramFile::read_with(self.assembly_file.clone(), source_manager.clone())? + .compile(&Debug::On, &libraries.libraries)?; let program_hash: [u8; 32] = program.hash().into(); println!("Debugging program with hash {}...", hex::encode(program_hash)); @@ -52,7 +54,8 @@ impl DebugCmd { // Instantiate DebugExecutor let mut debug_executor = - DebugExecutor::new(program, stack_inputs, advice_provider).map_err(Report::msg)?; + DebugExecutor::new(program, stack_inputs, advice_provider, source_manager) + .map_err(Report::msg)?; // build readline config let mut rl_config = Config::builder().auto_add_history(true); diff --git a/miden/src/repl/mod.rs b/miden/src/repl/mod.rs index b96b220928..591d841ce9 100644 --- a/miden/src/repl/mod.rs +++ b/miden/src/repl/mod.rs @@ -309,15 +309,16 @@ fn execute( let mut assembler = Assembler::default(); for library in provided_libraries { - assembler - .add_compiled_library(library.clone()) - .map_err(|err| format!("{err}"))?; + assembler.add_compiled_library(library).map_err(|err| format!("{err}"))?; } let program = assembler.assemble_program(program).map_err(|err| format!("{err}"))?; let stack_inputs = StackInputs::default(); - let host = DefaultHost::default(); + let mut host = DefaultHost::default(); + for library in provided_libraries { + host.load_mast_forest(library.mast_forest().clone()); + } let state_iter = processor::execute_iter(&program, stack_inputs, host); let (system, _, stack, chiplets, err) = state_iter.into_parts(); diff --git a/miden/src/tools/mod.rs b/miden/src/tools/mod.rs index 627f6df0e7..e93db16c31 100644 --- a/miden/src/tools/mod.rs +++ b/miden/src/tools/mod.rs @@ -35,7 +35,9 @@ impl Analyze { // fetch the stack and program inputs from the arguments let stack_inputs = input_data.parse_stack_inputs().map_err(Report::msg)?; - let host = DefaultHost::new(input_data.parse_advice_provider().map_err(Report::msg)?); + let mut host = DefaultHost::new(input_data.parse_advice_provider().map_err(Report::msg)?); + let stdlib = StdLibrary::default(); + host.load_mast_forest(stdlib.as_ref().mast_forest().clone()); let execution_details: ExecutionDetails = analyze(program.as_str(), stack_inputs, host) .expect("Could not retrieve execution details"); @@ -206,17 +208,14 @@ impl fmt::Display for ExecutionDetails { } /// Returns program analysis of a given program. -pub fn analyze( - program: &str, - stack_inputs: StackInputs, - host: H, -) -> Result +fn analyze(program: &str, stack_inputs: StackInputs, host: H) -> Result where H: Host, { + let stdlib = StdLibrary::default(); let program = Assembler::default() .with_debug_mode(true) - .with_compiled_library(StdLibrary::default())? + .with_compiled_library(&stdlib)? .assemble_program(program)?; let mut execution_details = ExecutionDetails::default(); diff --git a/miden/tests/integration/exec_iters.rs b/miden/tests/integration/exec_iters.rs index 43c30e7180..624025301c 100644 --- a/miden/tests/integration/exec_iters.rs +++ b/miden/tests/integration/exec_iters.rs @@ -1,6 +1,6 @@ use processor::{AsmOpInfo, ContextId, VmState}; use test_utils::{assert_eq, build_debug_test, Felt, ToElements, ONE}; -use vm_core::{AssemblyOp, Operation}; +use vm_core::{debuginfo::Location, AssemblyOp, Operation}; // EXEC ITER TESTS // ================================================================= @@ -12,10 +12,31 @@ fn test_exec_iter() { init_stack.push(i); }); let test = build_debug_test!(source, &init_stack); + let path = test.source.name(); let traces = test.execute_iter(); let fmp = Felt::new(2u64.pow(30)); let next_fmp = fmp + ONE; let mem = vec![(1_u64, slice_to_word(&[13, 14, 15, 16]))]; + let mem_storew1_loc = Some(Location { + path: path.clone(), + start: 33.into(), + end: (33 + 12).into(), + }); + let dropw_loc = Some(Location { + path: path.clone(), + start: 46.into(), + end: (46 + 5).into(), + }); + let push17_loc = Some(Location { + path: path.clone(), + start: 52.into(), + end: (52 + 7).into(), + }); + let locstore0_loc = Some(Location { + path: path.clone(), + start: 11.into(), + end: (11 + 11).into(), + }); let expected_states = vec![ VmState { clk: 0, @@ -49,7 +70,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Pad), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 3, "mem_storew.1".to_string(), false), + AssemblyOp::new( + mem_storew1_loc.clone(), + "#exec::#main".to_string(), + 3, + "mem_storew.1".to_string(), + false, + ), 1, )), stack: [0, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1].to_elements(), @@ -61,7 +88,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Incr), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 3, "mem_storew.1".to_string(), false), + AssemblyOp::new( + mem_storew1_loc.clone(), + "#exec::#main".to_string(), + 3, + "mem_storew.1".to_string(), + false, + ), 2, )), stack: [1, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2].to_elements(), @@ -73,7 +106,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::MStoreW), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 3, "mem_storew.1".to_string(), false), + AssemblyOp::new( + mem_storew1_loc, + "#exec::#main".to_string(), + 3, + "mem_storew.1".to_string(), + false, + ), 3, )), stack: [16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1].to_elements(), @@ -85,7 +124,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Drop), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 4, "dropw".to_string(), false), + AssemblyOp::new( + dropw_loc.clone(), + "#exec::#main".to_string(), + 4, + "dropw".to_string(), + false, + ), 1, )), stack: [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0].to_elements(), @@ -97,7 +142,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Drop), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 4, "dropw".to_string(), false), + AssemblyOp::new( + dropw_loc.clone(), + "#exec::#main".to_string(), + 4, + "dropw".to_string(), + false, + ), 2, )), stack: [14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0].to_elements(), @@ -109,7 +160,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Drop), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 4, "dropw".to_string(), false), + AssemblyOp::new( + dropw_loc.clone(), + "#exec::#main".to_string(), + 4, + "dropw".to_string(), + false, + ), 3, )), stack: [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0].to_elements(), @@ -121,7 +178,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Drop), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 4, "dropw".to_string(), false), + AssemblyOp::new( + dropw_loc, + "#exec::#main".to_string(), + 4, + "dropw".to_string(), + false, + ), 4, )), stack: [12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0, 0].to_elements(), @@ -133,7 +196,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Push(Felt::new(17))), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.17".to_string(), false), + AssemblyOp::new( + push17_loc, + "#exec::#main".to_string(), + 1, + "push.17".to_string(), + false, + ), 1, )), stack: [17, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0].to_elements(), @@ -190,7 +259,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Pad), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 4, "loc_store.0".to_string(), false), + AssemblyOp::new( + locstore0_loc.clone(), + "#exec::foo".to_string(), + 4, + "loc_store.0".to_string(), + false, + ), 1, )), stack: [0, 17, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0].to_elements(), @@ -202,7 +277,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::FmpAdd), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 4, "loc_store.0".to_string(), false), + AssemblyOp::new( + locstore0_loc.clone(), + "#exec::foo".to_string(), + 4, + "loc_store.0".to_string(), + false, + ), 2, )), stack: [2u64.pow(30) + 1, 17, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0] @@ -215,7 +296,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::MStore), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 4, "loc_store.0".to_string(), false), + AssemblyOp::new( + locstore0_loc.clone(), + "#exec::foo".to_string(), + 4, + "loc_store.0".to_string(), + false, + ), 3, )), stack: [17, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0].to_elements(), @@ -230,7 +317,13 @@ fn test_exec_iter() { ctx: ContextId::root(), op: Some(Operation::Drop), asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 4, "loc_store.0".to_string(), false), + AssemblyOp::new( + locstore0_loc.clone(), + "#exec::foo".to_string(), + 4, + "loc_store.0".to_string(), + false, + ), 4, )), stack: [12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0, 0].to_elements(), diff --git a/miden/tests/integration/flow_control/mod.rs b/miden/tests/integration/flow_control/mod.rs index ed34ff9b0a..966fe347d7 100644 --- a/miden/tests/integration/flow_control/mod.rs +++ b/miden/tests/integration/flow_control/mod.rs @@ -1,5 +1,6 @@ -use assembly::{ast::ModuleKind, Assembler, LibraryPath}; -use core::iter; +use alloc::sync::Arc; + +use assembly::{ast::ModuleKind, Assembler, LibraryPath, Report, SourceManager}; use miden_vm::Module; use processor::ExecutionError; use prover::Digest; @@ -203,11 +204,12 @@ fn simple_syscall() { end"; // TODO: update and use macro? - let test = Test { - kernel_source: Some(kernel_source.to_string()), - stack_inputs: StackInputs::try_from_ints([1, 2]).unwrap(), - ..Test::new(&format!("test{}", line!()), program_source, false) - }; + let mut test = Test::new(&format!("test{}", line!()), program_source, false); + test.stack_inputs = StackInputs::try_from_ints([1, 2]).unwrap(); + test.kernel_source = Some( + test.source_manager + .load(&format!("kernel{}", line!()), kernel_source.to_string()), + ); test.expect_stack(&[3]); test.prove_and_verify(vec![1, 2], false); @@ -390,7 +392,7 @@ fn simple_dyncall() { // ================================================================================================ #[test] -fn procref() { +fn procref() -> Result<(), Report> { let module_source = " use.std::math::u64 export.u64::overflowing_add @@ -402,12 +404,14 @@ fn procref() { // obtain procedures' MAST roots by compiling them as module let mast_roots: Vec = { + let source_manager = Arc::new(assembly::DefaultSourceManager::default()); let module_path = "test::foo".parse::().unwrap(); - let module = Module::parse_str(module_path, ModuleKind::Library, module_source).unwrap(); - let library = Assembler::default() + let mut parser = Module::parser(ModuleKind::Library); + let module = parser.parse_str(module_path, module_source, &source_manager)?; + let library = Assembler::new(source_manager) .with_compiled_library(StdLibrary::default()) .unwrap() - .assemble_library(iter::once(module)) + .assemble_library([module]) .unwrap(); let module_info = library.module_infos().next().unwrap(); @@ -444,4 +448,5 @@ fn procref() { ]); test.prove_and_verify(vec![], false); + Ok(()) } diff --git a/miden/tests/integration/operations/decorators/asmop.rs b/miden/tests/integration/operations/decorators/asmop.rs index 8dfcaba749..0270427f86 100644 --- a/miden/tests/integration/operations/decorators/asmop.rs +++ b/miden/tests/integration/operations/decorators/asmop.rs @@ -1,11 +1,27 @@ use processor::{AsmOpInfo, VmStateIterator}; use test_utils::{assert_eq, build_debug_test}; -use vm_core::{AssemblyOp, Felt, Operation}; +use vm_core::{debuginfo::Location, AssemblyOp, Felt, Operation}; #[test] fn asmop_one_span_block_test() { let source = "begin push.1 push.2 add end"; let test = build_debug_test!(source); + let path = test.source.name(); + let push1_loc = Some(Location { + path: path.clone(), + start: 6.into(), + end: (6 + 6).into(), + }); + let push2_loc = Some(Location { + path: path.clone(), + start: 13.into(), + end: (13 + 6).into(), + }); + let add_loc = Some(Location { + path: path.clone(), + start: 20.into(), + end: (20 + 3).into(), + }); let vm_state_iterator = test.execute_iter(); let expected_vm_state = vec![ VmStatePartial { @@ -21,7 +37,13 @@ fn asmop_one_span_block_test() { VmStatePartial { clk: 2, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 1, )), op: Some(Operation::Pad), @@ -29,7 +51,13 @@ fn asmop_one_span_block_test() { VmStatePartial { clk: 3, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc, + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 2, )), op: Some(Operation::Incr), @@ -37,7 +65,13 @@ fn asmop_one_span_block_test() { VmStatePartial { clk: 4, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.2".to_string(), false), + AssemblyOp::new( + push2_loc, + "#exec::#main".to_string(), + 1, + "push.2".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(2))), @@ -45,7 +79,7 @@ fn asmop_one_span_block_test() { VmStatePartial { clk: 5, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "add".to_string(), false), + AssemblyOp::new(add_loc, "#exec::#main".to_string(), 1, "add".to_string(), false), 1, )), op: Some(Operation::Add), @@ -64,6 +98,22 @@ fn asmop_one_span_block_test() { fn asmop_with_one_procedure() { let source = "proc.foo push.1 push.2 add end begin exec.foo end"; let test = build_debug_test!(source); + let path = test.source.name(); + let push1_loc = Some(Location { + path: path.clone(), + start: 9.into(), + end: (9 + 6).into(), + }); + let push2_loc = Some(Location { + path: path.clone(), + start: 16.into(), + end: (16 + 6).into(), + }); + let add_loc = Some(Location { + path: path.clone(), + start: 23.into(), + end: (23 + 3).into(), + }); let vm_state_iterator = test.execute_iter(); let expected_vm_state = vec![ VmStatePartial { @@ -79,7 +129,13 @@ fn asmop_with_one_procedure() { VmStatePartial { clk: 2, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::foo".to_string(), + 2, + "push.1".to_string(), + false, + ), 1, )), op: Some(Operation::Pad), @@ -87,7 +143,13 @@ fn asmop_with_one_procedure() { VmStatePartial { clk: 3, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc, + "#exec::foo".to_string(), + 2, + "push.1".to_string(), + false, + ), 2, )), op: Some(Operation::Incr), @@ -95,7 +157,13 @@ fn asmop_with_one_procedure() { VmStatePartial { clk: 4, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 1, "push.2".to_string(), false), + AssemblyOp::new( + push2_loc, + "#exec::foo".to_string(), + 1, + "push.2".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(2))), @@ -103,7 +171,7 @@ fn asmop_with_one_procedure() { VmStatePartial { clk: 5, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::foo".to_string(), 1, "add".to_string(), false), + AssemblyOp::new(add_loc, "#exec::foo".to_string(), 1, "add".to_string(), false), 1, )), op: Some(Operation::Add), @@ -126,6 +194,22 @@ fn asmop_repeat_test() { end end"; let test = build_debug_test!(source); + let path = test.source.name(); + let push1_loc = Some(Location { + path: path.clone(), + start: 43.into(), + end: (43 + 6).into(), + }); + let push2_loc = Some(Location { + path: path.clone(), + start: 50.into(), + end: (50 + 6).into(), + }); + let add_loc = Some(Location { + path: path.clone(), + start: 57.into(), + end: (57 + 3).into(), + }); let vm_state_iterator = test.execute_iter(); let expected_vm_state = vec![ VmStatePartial { @@ -151,7 +235,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 4, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 1, )), op: Some(Operation::Pad), @@ -159,7 +249,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 5, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 2, )), op: Some(Operation::Incr), @@ -167,7 +263,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 6, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.2".to_string(), false), + AssemblyOp::new( + push2_loc.clone(), + "#exec::#main".to_string(), + 1, + "push.2".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(2))), @@ -175,7 +277,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 7, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "add".to_string(), false), + AssemblyOp::new( + add_loc.clone(), + "#exec::#main".to_string(), + 1, + "add".to_string(), + false, + ), 1, )), op: Some(Operation::Add), @@ -194,7 +302,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 10, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 1, )), op: Some(Operation::Pad), @@ -202,7 +316,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 11, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 2, )), op: Some(Operation::Incr), @@ -210,7 +330,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 12, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.2".to_string(), false), + AssemblyOp::new( + push2_loc.clone(), + "#exec::#main".to_string(), + 1, + "push.2".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(2))), @@ -218,7 +344,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 13, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "add".to_string(), false), + AssemblyOp::new( + add_loc.clone(), + "#exec::#main".to_string(), + 1, + "add".to_string(), + false, + ), 1, )), op: Some(Operation::Add), @@ -243,7 +375,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 17, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 1, )), op: Some(Operation::Pad), @@ -251,7 +389,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 18, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc, + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 2, )), op: Some(Operation::Incr), @@ -259,7 +403,13 @@ fn asmop_repeat_test() { VmStatePartial { clk: 19, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.2".to_string(), false), + AssemblyOp::new( + push2_loc, + "#exec::#main".to_string(), + 1, + "push.2".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(2))), @@ -267,7 +417,7 @@ fn asmop_repeat_test() { VmStatePartial { clk: 20, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "add".to_string(), false), + AssemblyOp::new(add_loc, "#exec::#main".to_string(), 1, "add".to_string(), false), 1, )), op: Some(Operation::Add), @@ -302,6 +452,27 @@ fn asmop_conditional_execution_test() { //if branch let test = build_debug_test!(source, &[1, 1]); + let path = test.source.name(); + let eq_loc = Some(Location { + path: path.clone(), + start: 18.into(), + end: (18 + 2).into(), + }); + let push1_loc = Some(Location { + path: path.clone(), + start: 57.into(), + end: (57 + 6).into(), + }); + let push2_loc = Some(Location { + path: path.clone(), + start: 64.into(), + end: (64 + 6).into(), + }); + let add_loc = Some(Location { + path: path.clone(), + start: 71.into(), + end: (71 + 3).into(), + }); let vm_state_iterator = test.execute_iter(); let expected_vm_state = vec![ VmStatePartial { @@ -322,7 +493,7 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 3, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "eq".to_string(), false), + AssemblyOp::new(eq_loc, "#exec::#main".to_string(), 1, "eq".to_string(), false), 1, )), op: Some(Operation::Eq), @@ -345,7 +516,13 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 7, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc.clone(), + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 1, )), op: Some(Operation::Pad), @@ -353,7 +530,13 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 8, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 2, "push.1".to_string(), false), + AssemblyOp::new( + push1_loc, + "#exec::#main".to_string(), + 2, + "push.1".to_string(), + false, + ), 2, )), op: Some(Operation::Incr), @@ -361,7 +544,13 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 9, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.2".to_string(), false), + AssemblyOp::new( + push2_loc, + "#exec::#main".to_string(), + 1, + "push.2".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(2))), @@ -369,7 +558,7 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 10, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "add".to_string(), false), + AssemblyOp::new(add_loc, "#exec::#main".to_string(), 1, "add".to_string(), false), 1, )), op: Some(Operation::Add), @@ -395,6 +584,27 @@ fn asmop_conditional_execution_test() { //else branch let test = build_debug_test!(source, &[1, 0]); + let path = test.source.name(); + let eq_loc = Some(Location { + path: path.clone(), + start: 18.into(), + end: (18 + 2).into(), + }); + let push3_loc = Some(Location { + path: path.clone(), + start: 108.into(), + end: (108 + 6).into(), + }); + let push4_loc = Some(Location { + path: path.clone(), + start: 115.into(), + end: (115 + 6).into(), + }); + let add_loc = Some(Location { + path: path.clone(), + start: 122.into(), + end: (122 + 3).into(), + }); let vm_state_iterator = test.execute_iter(); let expected_vm_state = vec![ VmStatePartial { @@ -415,7 +625,7 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 3, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "eq".to_string(), false), + AssemblyOp::new(eq_loc, "#exec::#main".to_string(), 1, "eq".to_string(), false), 1, )), op: Some(Operation::Eq), @@ -438,7 +648,13 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 7, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.3".to_string(), false), + AssemblyOp::new( + push3_loc, + "#exec::#main".to_string(), + 1, + "push.3".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(3))), @@ -446,7 +662,13 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 8, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "push.4".to_string(), false), + AssemblyOp::new( + push4_loc, + "#exec::#main".to_string(), + 1, + "push.4".to_string(), + false, + ), 1, )), op: Some(Operation::Push(Felt::new(4))), @@ -454,7 +676,7 @@ fn asmop_conditional_execution_test() { VmStatePartial { clk: 9, asmop: Some(AsmOpInfo::new( - AssemblyOp::new("#exec::#main".to_string(), 1, "add".to_string(), false), + AssemblyOp::new(add_loc, "#exec::#main".to_string(), 1, "add".to_string(), false), 1, )), op: Some(Operation::Add), diff --git a/miden/tests/integration/operations/io_ops/env_ops.rs b/miden/tests/integration/operations/io_ops/env_ops.rs index 8744d9aafb..15048716ef 100644 --- a/miden/tests/integration/operations/io_ops/env_ops.rs +++ b/miden/tests/integration/operations/io_ops/env_ops.rs @@ -1,3 +1,4 @@ +use assembly::SourceManager; use processor::FMP_MIN; use test_utils::{build_op_test, build_test, StackInputs, Test, Word, STACK_TOP_SIZE}; use vm_core::{ @@ -144,11 +145,13 @@ fn caller() { end"; // TODO: update and use macro? - let test = Test { - kernel_source: Some(kernel_source.to_string()), - stack_inputs: StackInputs::try_from_ints([1, 2, 3, 4, 5]).unwrap(), - ..Test::new(&format!("test{}", line!()), program_source, false) - }; + let mut test = Test::new(&format!("test{}", line!()), program_source, false); + test.stack_inputs = StackInputs::try_from_ints([1, 2, 3, 4, 5]).unwrap(); + test.kernel_source = Some( + test.source_manager + .load(&format!("kernel{}", line!()), kernel_source.to_string()), + ); + // top 4 elements should be overwritten with the hash of `bar` procedure, but the 5th // element should remain untouched let bar_hash = build_bar_hash(); diff --git a/processor/src/debug.rs b/processor/src/debug.rs index 5ae7441d00..c891dce2ae 100644 --- a/processor/src/debug.rs +++ b/processor/src/debug.rs @@ -316,3 +316,10 @@ impl fmt::Display for AsmOpInfo { write!(f, "{}, cycles={}", self.asmop, self.cycle_idx) } } + +impl AsRef for AsmOpInfo { + #[inline] + fn as_ref(&self) -> &AssemblyOp { + &self.asmop + } +} diff --git a/processor/src/host/mod.rs b/processor/src/host/mod.rs index 86a2128092..d70622d270 100644 --- a/processor/src/host/mod.rs +++ b/processor/src/host/mod.rs @@ -285,6 +285,15 @@ pub struct DefaultHost { store: MemMastForestStore, } +impl Clone for DefaultHost { + fn clone(&self) -> Self { + Self { + adv_provider: self.adv_provider.clone(), + store: self.store.clone(), + } + } +} + impl Default for DefaultHost { fn default() -> Self { Self { diff --git a/stdlib/build.rs b/stdlib/build.rs index 42d03bfaa2..c251d735a7 100644 --- a/stdlib/build.rs +++ b/stdlib/build.rs @@ -4,7 +4,7 @@ use assembly::{ library::CompiledLibrary, LibraryNamespace, Version, }; -use std::{env, path::Path}; +use std::{env, path::Path, sync::Arc}; // CONSTANTS // ================================================================================================ @@ -26,10 +26,11 @@ fn main() -> Result<()> { let manifest_dir = env!("CARGO_MANIFEST_DIR"); let asm_dir = Path::new(manifest_dir).join(ASM_DIR_PATH); + let source_manager = Arc::new(assembly::DefaultSourceManager::default()); let namespace = "std".parse::().expect("invalid base namespace"); // TODO: Add version to `Library` let _version = env!("CARGO_PKG_VERSION").parse::().expect("invalid cargo version"); - let stdlib = CompiledLibrary::from_dir(asm_dir, namespace)?; + let stdlib = CompiledLibrary::from_dir(asm_dir, namespace, source_manager)?; // write the masl output let build_dir = env::var("OUT_DIR").unwrap(); diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index 4ae868ddb4..b6b1585bbe 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -27,10 +27,7 @@ use vm_core::{chiplets::hasher::apply_permutation, ProgramInfo}; // EXPORTS // ================================================================================================ -pub use assembly::{ - diagnostics::{Report, SourceFile}, - LibraryPath, -}; +pub use assembly::{diagnostics::Report, LibraryPath, SourceFile, SourceManager}; pub use pretty_assertions::{assert_eq, assert_ne, assert_str_eq}; pub use processor::{ AdviceInputs, AdviceProvider, ContextId, DefaultHost, ExecutionError, ExecutionOptions, @@ -176,8 +173,9 @@ macro_rules! assert_assembler_diagnostic { /// - Execution error test: check that running a program compiled from the given source causes an /// ExecutionError which contains the specified substring. pub struct Test { + pub source_manager: Arc, pub source: Arc, - pub kernel_source: Option, + pub kernel_source: Option>, pub stack_inputs: StackInputs, pub advice_inputs: AdviceInputs, pub in_debug_mode: bool, @@ -191,8 +189,11 @@ impl Test { /// Creates the simplest possible new test, with only a source string and no inputs. pub fn new(name: &str, source: &str, in_debug_mode: bool) -> Self { + let source_manager = Arc::new(assembly::DefaultSourceManager::default()); + let source = source_manager.load(name, source.to_string()); Test { - source: Arc::new(SourceFile::new(name, source.to_string())), + source_manager, + source, kernel_source: None, stack_inputs: StackInputs::default(), advice_inputs: AdviceInputs::default(), @@ -289,13 +290,17 @@ impl Test { pub fn compile(&self) -> Result<(Program, Option), Report> { use assembly::{ast::ModuleKind, Assembler, CompileOptions}; - let (assembler, compiled_kernel) = if let Some(kernel) = self.kernel_source.as_ref() { - let kernel_lib = Assembler::default().assemble_kernel(kernel).unwrap(); + let (assembler, compiled_kernel) = if let Some(kernel) = self.kernel_source.clone() { + let kernel_lib = + Assembler::new(self.source_manager.clone()).assemble_kernel(kernel).unwrap(); let compiled_kernel = kernel_lib.mast_forest().clone(); - (Assembler::with_kernel(kernel_lib), Some(compiled_kernel)) + ( + Assembler::with_kernel(self.source_manager.clone(), kernel_lib), + Some(compiled_kernel), + ) } else { - (Assembler::default(), None) + (Assembler::new(self.source_manager.clone()), None) }; let mut assembler = self .add_modules diff --git a/test-utils/src/test_builders.rs b/test-utils/src/test_builders.rs index 79a54088fd..644c8fbb01 100644 --- a/test-utils/src/test_builders.rs +++ b/test-utils/src/test_builders.rs @@ -82,16 +82,18 @@ macro_rules! build_test_by_mode { $crate::Test::new(&name, $source, $in_debug_mode) }}; ($in_debug_mode:expr, $source:expr, $stack_inputs:expr) => {{ + use ::assembly::SourceManager; + let stack_inputs: Vec = $stack_inputs.to_vec(); let stack_inputs = $crate::StackInputs::try_from_ints(stack_inputs).unwrap(); let advice_inputs = $crate::AdviceInputs::default(); let name = format!("test{}", line!()); + let source_manager = ::alloc::sync::Arc::new(::assembly::DefaultSourceManager::default()); + let source = source_manager.load(&name, ::alloc::string::String::from($source)); $crate::Test { - source: ::alloc::sync::Arc::new(::assembly::diagnostics::SourceFile::new( - name, - ::alloc::string::String::from($source), - )), + source_manager, + source, kernel_source: None, stack_inputs, advice_inputs, @@ -103,6 +105,8 @@ macro_rules! build_test_by_mode { ( $in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr ) => {{ + use ::assembly::SourceManager; + let stack_inputs: Vec = $stack_inputs.to_vec(); let stack_inputs = $crate::StackInputs::try_from_ints(stack_inputs).unwrap(); let stack_values: Vec = $advice_stack.to_vec(); @@ -112,12 +116,12 @@ macro_rules! build_test_by_mode { .unwrap() .with_merkle_store(store); let name = format!("test{}", line!()); + let source_manager = ::alloc::sync::Arc::new(::assembly::DefaultSourceManager::default()); + let source = source_manager.load(&name, ::alloc::string::String::from($source)); $crate::Test { - source: ::alloc::sync::Arc::new(::assembly::diagnostics::SourceFile::new( - name, - ::alloc::string::String::from($source), - )), + source_manager, + source, kernel_source: None, stack_inputs, advice_inputs, @@ -129,6 +133,8 @@ macro_rules! build_test_by_mode { ( $in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr, $advice_merkle_store:expr ) => {{ + use ::assembly::SourceManager; + let stack_inputs: Vec = $stack_inputs.to_vec(); let stack_inputs = $crate::StackInputs::try_from_ints(stack_inputs).unwrap(); let stack_values: Vec = $advice_stack.to_vec(); @@ -137,12 +143,12 @@ macro_rules! build_test_by_mode { .unwrap() .with_merkle_store($advice_merkle_store); let name = format!("test{}", line!()); + let source_manager = ::alloc::sync::Arc::new(::assembly::DefaultSourceManager::default()); + let source = source_manager.load(&name, ::alloc::string::String::from($source)); $crate::Test { - source: ::alloc::sync::Arc::new(::assembly::diagnostics::SourceFile::new( - name, - String::from($source), - )), + source_manager, + source, kernel_source: None, stack_inputs, advice_inputs, @@ -152,6 +158,8 @@ macro_rules! build_test_by_mode { } }}; ($in_debug_mode:expr, $source:expr, $stack_inputs:expr, $advice_stack:expr, $advice_merkle_store:expr, $advice_map:expr) => {{ + use ::assembly::SourceManager; + let stack_inputs: Vec = $stack_inputs.to_vec(); let stack_inputs = $crate::StackInputs::try_from_ints(stack_inputs).unwrap(); let stack_values: Vec = $advice_stack.to_vec(); @@ -161,12 +169,12 @@ macro_rules! build_test_by_mode { .with_merkle_store($advice_merkle_store) .with_map($advice_map); let name = format!("test{}", line!()); + let source_manager = ::alloc::sync::Arc::new(::assembly::DefaultSourceManager::default()); + let source = source_manager.load(&name, ::alloc::string::String::from($source)); $crate::Test { - source: ::alloc::sync::Arc::new(::assembly::diagnostics::SourceFile::new( - name, - ::alloc::string::String::from($source), - )), + source_manager, + source, kernel_source: None, stack_inputs, advice_inputs,