diff --git a/src/main.rs b/src/main.rs index 5ee91bd..7346551 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,7 +22,10 @@ fn main() -> Result<()> { } let mut tokenizer = Tokenizer::new(PathBuf::from(&path))?; - println!("{:?}", tokenizer.tokenize()?); + let tokens = tokenizer.tokenize()?; + for token in &tokens { + println!("{}: {:?}", token.error_prefix(&path), token); + } return Ok(()); } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 12e189c..158f492 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -4,12 +4,25 @@ use anyhow::Result; use crate::reader::{CharReader, CharType}; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum Token { - Text(String), - CommentMark, - HelpMark, - EqualSign, + Text(usize, usize, String), + CommentMark(usize, usize), + HelpMark(usize, usize), + EqualSign(usize, usize), +} + +impl Token { + pub fn error_prefix(&self, path: &String) -> String { + let (line, column) = match self { + Token::Text(x, y, _) => (x, y), + Token::CommentMark(x, y) => (x, y), + Token::HelpMark(x, y) => (x, y), + Token::EqualSign(x, y) => (x, y), + }; + + format!("{path}:{line}:{column}") + } } pub struct Tokenizer { @@ -23,6 +36,30 @@ impl Tokenizer { }) } + fn text(&self, buffer: String, eol: bool, prepends_help: bool) -> Token { + let adjust = match (eol, prepends_help) { + (true, false) => -1, + (false, true) => 2, + _ => 0, + } + (buffer.len() as i8); + + Token::Text( + self.reader.line, + self.reader.column - (adjust as usize), + buffer.trim().to_string(), + ) + } + + fn equal_sign(&self) -> Token { + Token::EqualSign(self.reader.line, self.reader.column) + } + fn comment_mark(&self) -> Token { + Token::CommentMark(self.reader.line, self.reader.column) + } + fn help_mark(&self) -> Token { + Token::HelpMark(self.reader.line, self.reader.column - 2) + } + fn next_tokens(&mut self) -> Result> { let mut buffer = "".to_string(); loop { @@ -33,23 +70,27 @@ impl Tokenizer { if buffer.is_empty() { continue; } - return Ok(vec![Token::Text(buffer.trim().to_string())]); + return Ok(vec![self.text(buffer, true, false)]); } CharType::Char(c) => { let mut token: Option = None; + let mut prepends_help = false; if c == '=' { - token = Some(Token::EqualSign); - } else if c == '#' && self.reader.column == 1 { - token = Some(Token::CommentMark); - } else if c == ' ' && buffer.ends_with(" #") { - buffer = buffer.strip_suffix(" #").unwrap_or("").to_string(); - token = Some(Token::HelpMark); + token = Some(self.equal_sign()); + } else if c == '#' { + if self.reader.column == 1 { + token = Some(self.comment_mark()); + } else if buffer.ends_with(" ") { + buffer = buffer.strip_suffix(" ").unwrap_or("").to_string(); + prepends_help = true; + token = Some(self.help_mark()); + } } if let Some(t) = token { if buffer.is_empty() { return Ok(vec![t]); } - return Ok(vec![Token::Text(buffer.trim().to_string()), t]); + return Ok(vec![self.text(buffer, false, prepends_help), t]); } buffer.push(c) } @@ -70,3 +111,64 @@ impl Tokenizer { Ok(tokens) } } + +// TODO: move to tests/ as integration test? +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tokenizer() { + let sample = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".env.sample"); + let mut tokenizer = Tokenizer::new(sample).unwrap(); + let tokens = tokenizer.tokenize().unwrap(); + assert_eq!(tokens.len(), 19); + + // line 1 + assert_eq!(tokens[0], Token::CommentMark(1, 1)); + assert_eq!(tokens[1], Token::Text(1, 2, "Createnv".to_string())); + assert_eq!(tokens[2], Token::CommentMark(2, 1)); + + // line 2 + assert_eq!( + tokens[3], + Token::Text( + 2, + 2, + "This is a simple example of how Createnv works".to_string() + ) + ); + + // line 3 + assert_eq!(tokens[4], Token::Text(3, 1, "NAME".to_string())); + assert_eq!(tokens[5], Token::EqualSign(3, 5)); + assert_eq!(tokens[6], Token::HelpMark(3, 6)); + assert_eq!( + tokens[7], + Token::Text(3, 9, "What's your name?".to_string()) + ); + + // line 4 + assert_eq!(tokens[8], Token::Text(4, 1, "GREETING".to_string())); + assert_eq!(tokens[9], Token::EqualSign(4, 9)); + assert_eq!(tokens[10], Token::Text(4, 10, "Hello, {NAME}!".to_string())); + + // line 5 + assert_eq!( + tokens[11], + Token::Text(5, 1, "DO_YOU_LIKE_OPEN_SOURCE".to_string()) + ); + assert_eq!(tokens[12], Token::EqualSign(5, 24)); + assert_eq!(tokens[13], Token::Text(5, 25, "True".to_string())); + assert_eq!(tokens[14], Token::HelpMark(5, 29)); + assert_eq!( + tokens[15], + Token::Text(5, 32, "Do you like open-source?".to_string()) + ); + + // line 6 + assert_eq!(tokens[16], Token::Text(6, 1, "PASSWORD".to_string())); + assert_eq!(tokens[17], Token::EqualSign(6, 9)); + assert_eq!(tokens[18], Token::Text(6, 10, "".to_string())); + } +}