Skip to content

Commit

Permalink
feat: add custom syntax support
Browse files Browse the repository at this point in the history
  • Loading branch information
@InioX
InioX committed Dec 25, 2024
1 parent 543dbd5 commit 4eac423
Show file tree
Hide file tree
Showing 7 changed files with 188 additions and 106 deletions.
4 changes: 2 additions & 2 deletions matugen-parser/src/errors/parse.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ impl ParseError<'_> {
ParseError {
err_type,
start: parser.last_bracket_start,
end: parser.prev_token_end,
end: parser.lexer_state.prev_token_end,
source: parser.source,
filename: &parser.filename,
line_number: parser.lexer.cur_line,
line_number: parser.lexer_state.lexer.cur_line,
}
}
}
Expand Down
12 changes: 7 additions & 5 deletions matugen-parser/src/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,21 @@ pub enum Kind {
String,
Number,
Float,
Lbracket,
RBracket,
Dot,
Bar,
Space,
Colon,
NewLine,
Eof,
Sof,
Identifier,

// SPECIAL TOKENS
LBracket,
RBracket,
LessThan,
GreaterThan,
Asterisk,
Bar,
Dot,
}

use std::str::FromStr;
Expand Down Expand Up @@ -119,7 +121,7 @@ impl<'a> Lexer<'a> {
}

match next_char.unwrap() {
'{' => (Kind::Lbracket, TokenValue::None),
'{' => (Kind::LBracket, TokenValue::None),
'}' => (Kind::RBracket, TokenValue::None),
':' => (Kind::Colon, TokenValue::None),
'<' => (Kind::LessThan, TokenValue::None),
Expand Down
6 changes: 4 additions & 2 deletions matugen-parser/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ use core::panic;
use std::path::PathBuf;

use lexer::Lexer;
use parser::Parser;
use parser::{settings::SyntaxSettings, Parser};

mod errors;
mod lexer;
Expand Down Expand Up @@ -44,7 +44,9 @@ fn main() {
// let file_path = "./matugen-parser/example/template.txt";
let src = std::fs::read_to_string(&file_path).unwrap();

let mut parser = Parser::new(&src, file_path_absolute);
let syntax = SyntaxSettings::new(['{', '{'], ['}', '}'], ['<', '*'], ['>', '*']);

let mut parser = Parser::new(&src, file_path_absolute, &syntax);

// parser.get_keywords();

Expand Down
103 changes: 57 additions & 46 deletions matugen-parser/src/parser/keywords.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,27 +13,28 @@ use super::Parser;

impl Parser<'_> {
fn get_opening(&mut self) -> Option<usize> {
let mut start = self.cur_token().start;
let mut start = self.lexer_state.cur_token().start;

self.bump_any();
self.lexer_state.bump_any();

while !self.opened {
if self.eat(Kind::Lbracket) {
if self.lexer_state.eat(&self.syntax.keyword_opening[0]) {
self.opened = true;
self.closed = false;
} else if self.eat(Kind::Eof) {
} else if self.lexer_state.eat(&Kind::Eof) {
return None;
}
self.bump_while_not(Kind::Lbracket);
start = self.cur_token().start;
self.lexer_state
.bump_while_not(&self.syntax.keyword_opening[1]);
start = self.lexer_state.cur_token().start;
}
Some(start)
}

pub fn get_closing(&mut self) -> Result<(), ParseError> {
println!("STARTING TO CLOSE");
self.bump_any();
if self.eat(Kind::RBracket) {
self.lexer_state.bump_any();
if self.lexer_state.eat(&self.syntax.keyword_closing[0]) {
self.closed = true;
self.opened = false;
Ok(())
Expand All @@ -48,15 +49,23 @@ impl Parser<'_> {
pub fn get_keywords(&mut self) -> Vec<Statement> {
let mut vec: Vec<Statement> = vec![];

while !self.at(Kind::Eof) {
if !self.at(Kind::Lbracket) {
self.bump_until_not_at(Kind::Lbracket);
while !self.lexer_state.at(&Kind::Eof) {
if !self.lexer_state.at(&self.syntax.keyword_opening[0]) {
self.lexer_state
.bump_until_not_at(&self.syntax.keyword_opening[0]);
}

// We would only get the second bracket at the start without the -1,
// the opening will ALWAYS have two brackets unlike the closing, which
// might have an error inside of it (so we dont look ahead for the closing).
self.last_bracket_start = self.get_opening().unwrap() - 1;

let opening = self.get_opening();

if opening.is_none() {
return vec;
}

self.last_bracket_start = opening.unwrap() - 1;
let start = self.start_node();

let mut strings: Vec<TokenValue> = vec![];
Expand All @@ -82,15 +91,15 @@ impl Parser<'_> {
fn get_filter(&mut self) -> Result<Option<FilterDefinition>, ParseError> {
let start = self.start_node();

self.bump_while_not(Kind::String);
self.lexer_state.bump_while_not(&Kind::String);

let name = self.cur_token().clone().value;
let name = self.lexer_state.cur_token().clone().value;

let mut filter_args: Vec<TokenValue> = vec![];

handle_error(self.collect_filter_args(&mut filter_args));

if self.at(Kind::RBracket) {
if self.lexer_state.at(&self.syntax.keyword_closing[0]) {
handle_error(self.get_closing());
return Ok(Some(FilterDefinition {
node: self.finish_node(start),
Expand All @@ -104,46 +113,47 @@ impl Parser<'_> {
filter_name: name,
arguments: filter_args,
}))
// self.bump_while_not(Kind::RBracket);
// self.lexer_state.bump_while_not(&Kind::RBracket);
}

fn collect_filter_args(
&mut self,
arguments: &mut Vec<TokenValue>,
) -> Result<Vec<TokenValue>, ParseError> {
// THIS SHOULD BE THE FILTER NAME
self.eat(Kind::String);
self.lexer_state.eat(&Kind::String);

if !self.eat_ignore_spaces(Kind::Colon) {
if !self.lexer_state.eat_ignore_spaces(&Kind::Colon) {
println!(
"{}",
format!("DOESNT HAVE ANY ARGS: {:?}", self.cur_token())
format!("DOESNT HAVE ANY ARGS: {:?}", self.lexer_state.cur_token())
.red()
.bold()
);
self.bump_while_not(Kind::RBracket)
self.lexer_state
.bump_while_not(&self.syntax.keyword_closing[0])
} else {
// while !self.at(Kind::RBracket) {
// match self.cur_kind() {
// Kind::String => arguments.push(&self.cur_token.value),
// while !self.lexer_state.at(Kind::RBracket) {
// match self.lexer_state.cur_kind() {
// Kind::String => arguments.push(&self.lexer_state.cur_token.value),
// Kind::Number => todo!(),
// _ => {}
// }
// }
loop {
match self.cur_kind() {
match self.lexer_state.cur_kind() {
Kind::Space => {
self.bump_until_not_at(Kind::Space);
self.lexer_state.bump_until_not_at(&Kind::Space);
}
Kind::String => {
arguments.push(self.cur_token.value.clone());
self.bump(Kind::String)
arguments.push(self.lexer_state.cur_token.value.clone());
self.lexer_state.bump(&Kind::String)
}
Kind::Number => {
arguments.push(self.cur_token.value.clone());
self.bump(Kind::Number)
arguments.push(self.lexer_state.cur_token.value.clone());
self.lexer_state.bump(&Kind::Number)
}
Kind::RBracket => {
kind if *kind == self.syntax.keyword_closing[1] => {
break;
}
_ => {
Expand Down Expand Up @@ -172,32 +182,33 @@ impl Parser<'_> {
filters: &mut Vec<FilterDefinition>,
) -> Result<(), ParseError> {
// Always first string, what comes after we cant know
self.bump_while_not(Kind::String);
strings.push(self.cur_val().clone());
self.lexer_state.bump_while_not(&Kind::String);
strings.push(self.lexer_state.cur_val().clone());

self.bump_any();
self.lexer_state.bump_any();

while !&self.closed && !self.at(Kind::Eof) {
match &self.cur_kind() {
while !&self.closed && !self.lexer_state.at(&Kind::Eof) {
match &self.lexer_state.cur_kind() {
Kind::Dot => {
if self.seen_dot && self.eat(Kind::Dot) {
if self.seen_dot && self.lexer_state.eat(&Kind::Dot) {
self.seen_dot = false;
return Err(ParseError::new_from_parser(
ParseErrorTypes::DoubleDot,
&self,
));
} else {
self.seen_dot = true;
self.bump(Kind::Dot);
self.lexer_state.bump(&Kind::Dot);
}
}
Kind::String => {
if self.seen_dot {
strings.push(self.cur_token.clone().value);
self.bump(Kind::String);
strings.push(self.lexer_state.cur_token.clone().value);
self.lexer_state.bump(&Kind::String);
self.seen_dot = false;
} else {
self.bump_while_not(Kind::RBracket);
self.lexer_state
.bump_while_not(&self.syntax.keyword_closing[0]);
return Err(ParseError::new_from_parser(
ParseErrorTypes::DoubleString,
&self,
Expand All @@ -215,9 +226,9 @@ impl Parser<'_> {
Err(e) => eprintln!("{}", e),
}
}
Kind::RBracket => {
kind if **kind == self.syntax.keyword_closing[0] => {
return self.get_closing();
// if self.eat(Kind::RBracket) {
// if self.lexer_state.eat(Kind::RBracket) {
// self.closed = true;
// self.opened = false;
// println!("closed without filter")
Expand All @@ -226,11 +237,11 @@ impl Parser<'_> {
// break;
// }
}
Kind::Space => self.bump(Kind::Space),
Kind::NewLine => self.bump(Kind::NewLine),
Kind::Identifier => self.bump(Kind::Identifier),
Kind::Space => self.lexer_state.bump(&Kind::Space),
Kind::NewLine => self.lexer_state.bump(&Kind::NewLine),
Kind::Identifier => self.lexer_state.bump(&Kind::Identifier),
_ => {
println!("{:?}", self.cur_token());
println!("{:?}", self.lexer_state.cur_token());
}
}
}
Expand Down
23 changes: 23 additions & 0 deletions matugen-parser/src/parser/language.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
use crate::lexer::Kind;

use super::Parser;

// impl Parser<'_> {
// pub fn get_opening_fn(&mut self) -> Option<usize> {
// let mut start = self.cur_token().start;

// self.bump_any();

// while !self.opened {
// if self.eat(Kind::LessThan) {
// self.opened = true;
// self.closed = false;
// } else if self.eat(Kind::Eof) {
// return None;
// }
// self.bump_while_not(Kind::LessThan);
// start = self.cur_token().start;
// }
// Some(start)
// }
// }
Loading

0 comments on commit 4eac423

Please sign in to comment.