From 3c4d42a797a72fc09d66be89b2dc925ecb4c37e8 Mon Sep 17 00:00:00 2001 From: Rex Magana Date: Fri, 17 May 2024 17:51:09 -0700 Subject: [PATCH] working test --- crates/compiler/src/compilation_steps.rs | 2 +- .../add_initial_value_registrations.rs | 2 +- .../add_tracking_declarations.rs | 4 +- .../src/compilation_steps/check_types.rs | 2 +- .../compilation_steps/clean_up_diagnostics.rs | 2 +- .../create_declarations_for_tracking_nodes.rs | 2 +- .../src/compilation_steps/early_breaks.rs | 4 +- .../compilation_steps/find_tracking_nodes.rs | 2 +- .../src/compilation_steps/generate_code.rs | 2 +- .../src/compilation_steps/get_declarations.rs | 2 +- .../src/compilation_steps/parse_files.rs | 2 +- .../register_initial_variables.rs | 4 +- .../src/compilation_steps/register_strings.rs | 2 +- .../resolve_deferred_type_diagnostic.rs | 2 +- .../validate_unique_node_names.rs | 4 +- crates/compiler/src/compiler.rs | 34 +++++++++- .../compiler/src/compiler/run_compilation.rs | 54 ++++++--------- crates/compiler/src/file_parse_result.rs | 11 +++- crates/compiler/src/lib.rs | 3 +- crates/compiler/src/string_table_manager.rs | 2 +- crates/yarnspinner/tests/language_tests.rs | 65 +++++++++++++++++++ 21 files changed, 145 insertions(+), 62 deletions(-) diff --git a/crates/compiler/src/compilation_steps.rs b/crates/compiler/src/compilation_steps.rs index 70525273..4e29eb99 100644 --- a/crates/compiler/src/compilation_steps.rs +++ b/crates/compiler/src/compilation_steps.rs @@ -13,7 +13,7 @@ mod register_strings; mod resolve_deferred_type_diagnostic; mod validate_unique_node_names; -pub(crate) use self::{ +pub use self::{ add_initial_value_registrations::*, add_tracking_declarations::*, check_types::*, clean_up_diagnostics::*, create_declarations_for_tracking_nodes::*, early_breaks::*, find_tracking_nodes::*, generate_code::*, get_declarations::*, parse_files::*, diff --git a/crates/compiler/src/compilation_steps/add_initial_value_registrations.rs b/crates/compiler/src/compilation_steps/add_initial_value_registrations.rs index 5a1a16dd..90dee11a 100644 --- a/crates/compiler/src/compilation_steps/add_initial_value_registrations.rs +++ b/crates/compiler/src/compilation_steps/add_initial_value_registrations.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use yarnspinner_core::prelude::*; use yarnspinner_core::types::{Type, TypeFormat}; -pub(crate) fn add_initial_value_registrations( +pub fn add_initial_value_registrations( mut state: CompilationIntermediate, ) -> CompilationIntermediate { // Last step: take every variable declaration we found in all diff --git a/crates/compiler/src/compilation_steps/add_tracking_declarations.rs b/crates/compiler/src/compilation_steps/add_tracking_declarations.rs index 189a44d3..b398bd7b 100644 --- a/crates/compiler/src/compilation_steps/add_tracking_declarations.rs +++ b/crates/compiler/src/compilation_steps/add_tracking_declarations.rs @@ -2,9 +2,7 @@ use crate::prelude::*; use yarnspinner_core::prelude::*; use yarnspinner_core::types::Type; -pub(crate) fn add_tracking_declarations( - mut state: CompilationIntermediate, -) -> CompilationIntermediate { +pub fn add_tracking_declarations(mut state: CompilationIntermediate) -> CompilationIntermediate { let tracking_declarations: Vec<_> = state .tracking_nodes .iter() diff --git a/crates/compiler/src/compilation_steps/check_types.rs b/crates/compiler/src/compilation_steps/check_types.rs index 5c0b32c8..e96f4a3b 100644 --- a/crates/compiler/src/compilation_steps/check_types.rs +++ b/crates/compiler/src/compilation_steps/check_types.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use crate::visitors::TypeCheckVisitor; use antlr_rust::tree::ParseTreeVisitorCompat; -pub(crate) fn check_types(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn check_types(mut state: CompilationIntermediate) -> CompilationIntermediate { for file in &state.parsed_files { let mut visitor = TypeCheckVisitor::new(state.known_variable_declarations.clone(), file.clone()); diff --git a/crates/compiler/src/compilation_steps/clean_up_diagnostics.rs b/crates/compiler/src/compilation_steps/clean_up_diagnostics.rs index f3203c99..0117929e 100644 --- a/crates/compiler/src/compilation_steps/clean_up_diagnostics.rs +++ b/crates/compiler/src/compilation_steps/clean_up_diagnostics.rs @@ -2,7 +2,7 @@ use crate::listeners::DiagnosticVec; use crate::prelude::*; use std::collections::HashSet; -pub(crate) fn clean_up_diagnostics(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn clean_up_diagnostics(mut state: CompilationIntermediate) -> CompilationIntermediate { let total_diagnostics = if let Some(Ok(compilation)) = state.result.as_ref() { compilation .warnings diff --git a/crates/compiler/src/compilation_steps/create_declarations_for_tracking_nodes.rs b/crates/compiler/src/compilation_steps/create_declarations_for_tracking_nodes.rs index 44c9447d..4005319c 100644 --- a/crates/compiler/src/compilation_steps/create_declarations_for_tracking_nodes.rs +++ b/crates/compiler/src/compilation_steps/create_declarations_for_tracking_nodes.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use yarnspinner_core::prelude::Library; use yarnspinner_core::types::Type; -pub(crate) fn create_declarations_for_tracking_nodes( +pub fn create_declarations_for_tracking_nodes( mut state: CompilationIntermediate, ) -> CompilationIntermediate { let tracking_declarations = state.tracking_nodes.iter().map(|node| { diff --git a/crates/compiler/src/compilation_steps/early_breaks.rs b/crates/compiler/src/compilation_steps/early_breaks.rs index 901f4c00..e7ec56e6 100644 --- a/crates/compiler/src/compilation_steps/early_breaks.rs +++ b/crates/compiler/src/compilation_steps/early_breaks.rs @@ -1,6 +1,6 @@ use crate::prelude::*; -pub(crate) fn break_on_job_with_only_strings( +pub fn break_on_job_with_only_strings( mut state: CompilationIntermediate, ) -> CompilationIntermediate { if state.job.compilation_type == CompilationType::StringsOnly { @@ -15,7 +15,7 @@ pub(crate) fn break_on_job_with_only_strings( state } -pub(crate) fn break_on_job_with_only_declarations( +pub fn break_on_job_with_only_declarations( mut state: CompilationIntermediate, ) -> CompilationIntermediate { if state.job.compilation_type == CompilationType::DeclarationsOnly { diff --git a/crates/compiler/src/compilation_steps/find_tracking_nodes.rs b/crates/compiler/src/compilation_steps/find_tracking_nodes.rs index e82617b8..1fda56cc 100644 --- a/crates/compiler/src/compilation_steps/find_tracking_nodes.rs +++ b/crates/compiler/src/compilation_steps/find_tracking_nodes.rs @@ -3,7 +3,7 @@ use crate::visitors::NodeTrackingVisitor; use antlr_rust::tree::ParseTreeVisitorCompat; use std::collections::HashSet; -pub(crate) fn find_tracking_nodes(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn find_tracking_nodes(mut state: CompilationIntermediate) -> CompilationIntermediate { // determining the nodes we need to track visits on // this needs to be done before we finish up with declarations // so that any tracking variables are included in the compiled declarations diff --git a/crates/compiler/src/compilation_steps/generate_code.rs b/crates/compiler/src/compilation_steps/generate_code.rs index afa88c8b..bf3eb7c6 100644 --- a/crates/compiler/src/compilation_steps/generate_code.rs +++ b/crates/compiler/src/compilation_steps/generate_code.rs @@ -5,7 +5,7 @@ use crate::visitors::KnownTypes; use crate::Result; use std::collections::{HashMap, HashSet}; -pub(crate) fn generate_code(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn generate_code(mut state: CompilationIntermediate) -> CompilationIntermediate { let has_errors = state.diagnostics.has_errors(); let results: Vec<_> = if has_errors { // We have errors, so we can't safely generate code. diff --git a/crates/compiler/src/compilation_steps/get_declarations.rs b/crates/compiler/src/compilation_steps/get_declarations.rs index ddf8c80d..af06c7ce 100644 --- a/crates/compiler/src/compilation_steps/get_declarations.rs +++ b/crates/compiler/src/compilation_steps/get_declarations.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use crate::visitors::DeclarationVisitor; use antlr_rust::tree::ParseTreeVisitorCompat; -pub(crate) fn get_declarations(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn get_declarations(mut state: CompilationIntermediate) -> CompilationIntermediate { // Find the variable declarations in these files. for file in &state.parsed_files { let mut variable_declaration_visitor = diff --git a/crates/compiler/src/compilation_steps/parse_files.rs b/crates/compiler/src/compilation_steps/parse_files.rs index 508fde6c..78a445c4 100644 --- a/crates/compiler/src/compilation_steps/parse_files.rs +++ b/crates/compiler/src/compilation_steps/parse_files.rs @@ -1,6 +1,6 @@ use crate::prelude::*; -pub(crate) fn parse_files(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn parse_files(mut state: CompilationIntermediate) -> CompilationIntermediate { for (file, chars) in state.job.files.iter().zip(state.file_chars.iter()) { let parse_result = parse_syntax_tree(file, chars, &mut state.diagnostics); state.parsed_files.push(parse_result); diff --git a/crates/compiler/src/compilation_steps/register_initial_variables.rs b/crates/compiler/src/compilation_steps/register_initial_variables.rs index 483d490f..939753a8 100644 --- a/crates/compiler/src/compilation_steps/register_initial_variables.rs +++ b/crates/compiler/src/compilation_steps/register_initial_variables.rs @@ -1,9 +1,7 @@ use crate::prelude::*; use yarnspinner_core::prelude::*; -pub(crate) fn register_initial_variables( - mut state: CompilationIntermediate, -) -> CompilationIntermediate { +pub fn register_initial_variables(mut state: CompilationIntermediate) -> CompilationIntermediate { let variables = &mut state.known_variable_declarations; let job_variable_declarations = state.job.variable_declarations.clone(); variables.extend(job_variable_declarations); diff --git a/crates/compiler/src/compilation_steps/register_strings.rs b/crates/compiler/src/compilation_steps/register_strings.rs index 906afab0..e6b70794 100644 --- a/crates/compiler/src/compilation_steps/register_strings.rs +++ b/crates/compiler/src/compilation_steps/register_strings.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use crate::visitors::{LastLineBeforeOptionsVisitor, StringTableGeneratorVisitor}; use antlr_rust::tree::ParseTreeVisitorCompat; -pub(crate) fn register_strings(mut state: CompilationIntermediate) -> CompilationIntermediate { +pub fn register_strings(mut state: CompilationIntermediate) -> CompilationIntermediate { // First pass: parse all files, generate their syntax trees, // and figure out what variables they've declared for file in &state.parsed_files { diff --git a/crates/compiler/src/compilation_steps/resolve_deferred_type_diagnostic.rs b/crates/compiler/src/compilation_steps/resolve_deferred_type_diagnostic.rs index 85579563..031e5e44 100644 --- a/crates/compiler/src/compilation_steps/resolve_deferred_type_diagnostic.rs +++ b/crates/compiler/src/compilation_steps/resolve_deferred_type_diagnostic.rs @@ -1,7 +1,7 @@ use crate::prelude::*; use std::collections::HashSet; -pub(crate) fn resolve_deferred_type_diagnostic( +pub fn resolve_deferred_type_diagnostic( mut state: CompilationIntermediate, ) -> CompilationIntermediate { let known_declarations: HashSet<_> = state diff --git a/crates/compiler/src/compilation_steps/validate_unique_node_names.rs b/crates/compiler/src/compilation_steps/validate_unique_node_names.rs index b228822d..9f5e8653 100644 --- a/crates/compiler/src/compilation_steps/validate_unique_node_names.rs +++ b/crates/compiler/src/compilation_steps/validate_unique_node_names.rs @@ -3,9 +3,7 @@ use crate::prelude::*; use antlr_rust::token::Token; use std::collections::HashMap; -pub(crate) fn validate_unique_node_names( - mut state: CompilationIntermediate, -) -> CompilationIntermediate { +pub fn validate_unique_node_names(mut state: CompilationIntermediate) -> CompilationIntermediate { // Ensure that all nodes names in this compilation are unique. Node // name uniqueness is important for several processes, so we do this // check here. diff --git a/crates/compiler/src/compiler.rs b/crates/compiler/src/compiler.rs index 7bfe378f..7e9e0ca6 100644 --- a/crates/compiler/src/compiler.rs +++ b/crates/compiler/src/compiler.rs @@ -1,13 +1,13 @@ //! Adapted from //! and -use crate::prelude::*; +use crate::{compilation_steps::*, prelude::*}; use std::path::Path; use yarnspinner_core::prelude::*; mod add_tags_to_lines; pub(crate) mod antlr_rust_ext; -pub(crate) mod run_compilation; +pub mod run_compilation; pub(crate) mod utils; #[allow(missing_docs)] @@ -99,7 +99,35 @@ impl Compiler { /// Compiles the Yarn files previously added into a [`Compilation`]. pub fn compile(&self) -> Result { - run_compilation::compile(self) + run_compilation::compile(self, Self::default_compiler_steps()) + } + + /// Compiles the Yarn files previously added into a [`Compilation`] with custom compilation steps. + pub fn compile_with_custom_steps( + &self, + custom_steps: Vec<&CompilationStep>, + ) -> Result { + run_compilation::compile(self, custom_steps) + } + + /// The default compilation steps that the compiler will use. + pub fn default_compiler_steps() -> Vec<&'static CompilationStep> { + vec![ + ®ister_initial_variables, + &parse_files, + ®ister_strings, + &validate_unique_node_names, + &break_on_job_with_only_strings, + &get_declarations, + &check_types, + &find_tracking_nodes, + &create_declarations_for_tracking_nodes, + &add_tracking_declarations, + &resolve_deferred_type_diagnostic, + &break_on_job_with_only_declarations, + &generate_code, + &add_initial_value_registrations, + ] } } diff --git a/crates/compiler/src/compiler/run_compilation.rs b/crates/compiler/src/compiler/run_compilation.rs index 66cc08fb..09214919 100644 --- a/crates/compiler/src/compiler/run_compilation.rs +++ b/crates/compiler/src/compiler/run_compilation.rs @@ -1,3 +1,5 @@ +//! The main entry point for running a compilation job. + use crate::compilation_steps::*; use crate::output::*; use crate::prelude::*; @@ -7,24 +9,7 @@ use crate::Result; use std::collections::{HashMap, HashSet}; /// Compile Yarn code, as specified by a compilation job. -pub(crate) fn compile(compiler: &Compiler) -> Result { - let compiler_steps: Vec<&CompilationStep> = vec![ - ®ister_initial_variables, - &parse_files, - ®ister_strings, - &validate_unique_node_names, - &break_on_job_with_only_strings, - &get_declarations, - &check_types, - &find_tracking_nodes, - &create_declarations_for_tracking_nodes, - &add_tracking_declarations, - &resolve_deferred_type_diagnostic, - &break_on_job_with_only_declarations, - &generate_code, - &add_initial_value_registrations, - ]; - +pub(crate) fn compile(compiler: &Compiler, steps: Vec<&CompilationStep>) -> Result { let chars: Vec> = compiler .files .iter() @@ -41,7 +26,7 @@ pub(crate) fn compile(compiler: &Compiler) -> Result { .collect(); let chars: Vec<_> = chars.iter().map(|c| c.as_slice()).collect(); let initial = CompilationIntermediate::from_job(compiler, chars); - let intermediate = compiler_steps.into_iter().fold(initial, |state, step| { + let intermediate = steps.into_iter().fold(initial, |state, step| { if state.early_break { state } else { @@ -55,24 +40,25 @@ pub(crate) fn compile(compiler: &Compiler) -> Result { result } -type CompilationStep = dyn Fn(CompilationIntermediate) -> CompilationIntermediate; +pub type CompilationStep = dyn Fn(CompilationIntermediate) -> CompilationIntermediate; -pub(crate) struct CompilationIntermediate<'input> { - pub(crate) job: &'input Compiler, - pub(crate) file_chars: Vec<&'input [u32]>, - pub(crate) result: Option>, +#[derive(Debug)] +pub struct CompilationIntermediate<'input> { + pub job: &'input Compiler, + pub file_chars: Vec<&'input [u32]>, + pub result: Option>, /// All variable declarations that we've encountered, PLUS the ones we knew about before - pub(crate) known_variable_declarations: Vec, + pub known_variable_declarations: Vec, /// All variable declarations that we've encountered during this compilation job - pub(crate) derived_variable_declarations: Vec, - pub(crate) potential_issues: Vec, - pub(crate) parsed_files: Vec>, - pub(crate) tracking_nodes: HashSet, - pub(crate) string_table: StringTableManager, - pub(crate) diagnostics: Vec, - pub(crate) file_tags: HashMap>, - pub(crate) known_types: KnownTypes, - pub(crate) early_break: bool, + pub derived_variable_declarations: Vec, + pub potential_issues: Vec, + pub parsed_files: Vec>, + pub tracking_nodes: HashSet, + pub string_table: StringTableManager, + pub diagnostics: Vec, + pub file_tags: HashMap>, + pub known_types: KnownTypes, + pub early_break: bool, } impl<'input> CompilationIntermediate<'input> { diff --git a/crates/compiler/src/file_parse_result.rs b/crates/compiler/src/file_parse_result.rs index 1aa5b104..9fb4cb4d 100644 --- a/crates/compiler/src/file_parse_result.rs +++ b/crates/compiler/src/file_parse_result.rs @@ -1,7 +1,7 @@ //! Adapted from use crate::prelude::{generated::yarnspinnerparser::*, *}; -use std::rc::Rc; +use std::{fmt::Formatter, rc::Rc}; /// Contains the result of parsing a single file of source code. /// @@ -34,3 +34,12 @@ impl<'input> FileParseResult<'input> { &self.parser.input } } + +impl<'input> std::fmt::Debug for FileParseResult<'input> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FileParseResult") + .field("name", &self.name) + .field("tree", &self.tree) + .finish() + } +} diff --git a/crates/compiler/src/lib.rs b/crates/compiler/src/lib.rs index cbd8d9a3..d7aed7f1 100644 --- a/crates/compiler/src/lib.rs +++ b/crates/compiler/src/lib.rs @@ -9,7 +9,7 @@ #![warn(missing_docs, missing_debug_implementations)] mod collections; -pub(crate) mod compilation_steps; +pub mod compilation_steps; pub(crate) mod compiler; pub(crate) mod error_strategy; mod file_parse_result; @@ -21,6 +21,7 @@ mod string_table_manager; pub(crate) mod token_ext; pub(crate) mod visitors; +pub use crate::compiler::run_compilation; pub use crate::compiler::Result; pub mod prelude { diff --git a/crates/compiler/src/string_table_manager.rs b/crates/compiler/src/string_table_manager.rs index 498b6b6e..1f5c1a8e 100644 --- a/crates/compiler/src/string_table_manager.rs +++ b/crates/compiler/src/string_table_manager.rs @@ -6,7 +6,7 @@ use std::ops::{Deref, DerefMut}; use yarnspinner_core::prelude::*; #[derive(Debug, Clone, Default)] -pub(crate) struct StringTableManager(pub HashMap); +pub struct StringTableManager(pub HashMap); impl StringTableManager { pub(crate) fn contains_implicit_string_tags(&self) -> bool { diff --git a/crates/yarnspinner/tests/language_tests.rs b/crates/yarnspinner/tests/language_tests.rs index 7ae493bb..112fb3b9 100644 --- a/crates/yarnspinner/tests/language_tests.rs +++ b/crates/yarnspinner/tests/language_tests.rs @@ -10,6 +10,8 @@ use test_base::prelude::*; use yarnspinner::compiler::*; use yarnspinner::core::*; use yarnspinner::runtime::*; +use yarnspinner_compiler::compilation_steps::*; +use yarnspinner_compiler::run_compilation::CompilationIntermediate; mod test_base; @@ -220,3 +222,66 @@ fn crashes_on_command_expression_evaluating_whitespace() { .with_compilation(result) .run_standard_testcase(); } + +#[test] +fn test_compile_with_custom_steps() { + let source = " + <> + <> + <> + "; + + let custom_steps: Vec<&dyn Fn(CompilationIntermediate) -> CompilationIntermediate> = vec![ + ®ister_initial_variables, + &parse_files, + ®ister_strings, + &|state| { + // Custom step: Update the string table + let mut string_table = state.string_table; + string_table.insert( + "custom_line_id".into(), + StringInfo { + text: "Custom line".to_string(), + node_name: "Start".to_string(), + line_number: 1, + file_name: "".to_string(), + is_implicit_tag: false, + metadata: vec![], + }, + ); + CompilationIntermediate { + string_table, + ..state + } + }, + &validate_unique_node_names, + &break_on_job_with_only_strings, + &get_declarations, + &check_types, + &find_tracking_nodes, + &create_declarations_for_tracking_nodes, + &add_tracking_declarations, + &resolve_deferred_type_diagnostic, + &break_on_job_with_only_declarations, + &generate_code, + &add_initial_value_registrations, + ]; + + let result = Compiler::from_test_source(source) + .compile_with_custom_steps(custom_steps) + .unwrap(); + + // Check that the custom line is present in the string table + assert!(result.string_table.contains_key(&"custom_line_id".into())); + assert_eq!( + result.string_table.get(&"custom_line_id".into()).unwrap(), + &StringInfo { + text: "Custom line".to_string(), + node_name: "Start".to_string(), + line_number: 1, + file_name: "".to_string(), + is_implicit_tag: false, + metadata: vec![], + } + ); +}