diff --git a/src/api/error/mod.rs b/src/api/error/mod.rs new file mode 100644 index 0000000..9128898 --- /dev/null +++ b/src/api/error/mod.rs @@ -0,0 +1,8 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +mod syntax; + +pub use self::syntax::{ SyntaxError, SyntaxErrorCode, UnexpectedToken }; diff --git a/src/api/error/syntax.rs b/src/api/error/syntax.rs new file mode 100644 index 0000000..348073b --- /dev/null +++ b/src/api/error/syntax.rs @@ -0,0 +1,44 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use api::Position; +use api::tokens::{ Token }; + +#[derive(Debug)] +pub struct SyntaxError { + pub code: SyntaxErrorCode, + pub starts_at: Position, + pub ends_at: Option, +} + +impl SyntaxError { + /// Specify the location where the error ends. + pub fn ends_at(mut self, pos: Position) -> SyntaxError { + self.ends_at = Some(pos); + self + } +} + +#[derive(Debug)] +pub enum SyntaxErrorCode { + ExpectedTokenButReceived { expected: Token, received: UnexpectedToken }, +} + +impl SyntaxErrorCode { + /// Specify the location where the error starts. + pub fn starts_at(self, pos: Position) -> SyntaxError { + SyntaxError { + code: self, + starts_at: pos, + ends_at: None, + } + } +} + +#[derive(Clone, Debug)] +pub enum UnexpectedToken { + Token(Token), + EndOfStream, +} diff --git a/src/api/lexer/error.rs b/src/api/lexer/error.rs new file mode 100644 index 0000000..727b794 --- /dev/null +++ b/src/api/lexer/error.rs @@ -0,0 +1,28 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use std::io; +use api::error::SyntaxError; +use std::result; + +#[derive(Debug)] +pub enum LexingError { + Syntax(SyntaxError), + Io(io::Error) +} + +impl From for LexingError { + fn from(other: io::Error) -> LexingError { + LexingError::Io(other) + } +} + +impl From for LexingError { + fn from(other: SyntaxError) -> LexingError { + LexingError::Syntax(other) + } +} + +pub type LexingResult = result::Result; diff --git a/src/api/lexer/lexing.rs b/src/api/lexer/lexing.rs new file mode 100644 index 0000000..a70c31b --- /dev/null +++ b/src/api/lexer/lexing.rs @@ -0,0 +1,83 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use api::Position; +use api::tokens::TokenRef; + +use api::lexer::{ Lexer, LexingResult }; +use api::error::{ SyntaxErrorCode, UnexpectedToken }; + +/// TokenRef wrapper for `Lexer` that additionaly has position. +#[derive(Debug)] +pub struct ItemRef<'t> { + pub token: TokenRef<'t>, + pub position: Position, +} + +/// Lexer token iterator. +/// +/// 'i is iteration lifetime, or "one use of lexer". +/// 't is template lifetime. It will live longer than this iteration. +pub struct Tokens<'i, 't> { + /// Position of the next token to get. + next_pos: Position, // temporary field until I get cursor in. + env: &'i Lexer, + code: &'t str, +} + +impl<'i, 't> Iterator for Tokens<'i, 't> { + type Item = LexingResult>; + + fn next(&mut self) -> Option>> { + // Hello, my name is Lexer. Twig Lexer. + // I am not very complicated. + match self.next_pos { + Position { line: 1, .. } => { + self.next_pos.line = 2; + Some(Ok(ItemRef { token: TokenRef::BlockStart, position: self.next_pos })) + }, + Position { line: 2, .. } => { + self.next_pos.line = 3; + Some(Ok(ItemRef { token: TokenRef::Name("§"), position: self.next_pos })) + }, + _ => None + } + } +} + +impl<'i, 't> Tokens<'i, 't> { + pub fn new<'ii, 'tt>(lexer: &'ii Lexer, code: &'tt str) -> Tokens<'ii, 'tt> { + Tokens { + next_pos: Position { line: 1, column: 1 }, + env: lexer, + code: code, + } + } + + pub fn expect(&mut self, expected: TokenRef<'t>) -> LexingResult> { + let maybe_item = self.next(); + match maybe_item { + None => Err( + SyntaxErrorCode::ExpectedTokenButReceived { + expected: expected.into(), + received: UnexpectedToken::EndOfStream + }.starts_at(self.next_pos).into() + ), + Some(Ok(item)) => { + if item.token == expected { + Ok(item.token) + } else { + Err( + SyntaxErrorCode::ExpectedTokenButReceived { + expected: expected.into(), + received: UnexpectedToken::Token(item.token.into()) + }.starts_at(item.position).into() + ) + } + }, + Some(Err(error)) => Err(error), + } + } +} diff --git a/src/api/lexer/mod.rs b/src/api/lexer/mod.rs new file mode 100644 index 0000000..0e46f27 --- /dev/null +++ b/src/api/lexer/mod.rs @@ -0,0 +1,54 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +/*! +Twig lexer. + +Produces a token stream from source template. + +# Summary + +This module is capable of taking a Twig input template, for example, this one: + +```twig +Hello +{% if world %} + world +{% else %} + {{ other }} +{% endif %} +``` + +And chopping it into tokens like these: + +TODO: Example code for this. +*/ + +mod lexing; +mod error; + +pub use self::lexing::{ Tokens, ItemRef }; +pub use self::error::{ LexingError, LexingResult }; + +#[derive(Copy, Clone)] +pub struct Options; + +impl Options { + pub fn default() -> Options { Options } +} + +pub struct Lexer; + +impl Lexer { + /// Create a new lexer from options and operator list. + pub fn new(options: Options, operators: Vec<&'static str>) -> Lexer { + Lexer + } + + /// Get a lexed stream of tokens. + pub fn tokens<'i, 't>(&'i self, code: &'t str) -> Tokens<'i, 't> { + Tokens::new(self, code) + } +} diff --git a/src/api/mod.rs b/src/api/mod.rs new file mode 100644 index 0000000..49f46cc --- /dev/null +++ b/src/api/mod.rs @@ -0,0 +1,19 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +/*! +Twig extension writer's API. +*/ + +pub mod tokens; +pub mod lexer; +pub mod error; + +/// Line-column position in a file. +#[derive(Debug, Default, Copy, Clone)] +pub struct Position { + pub line: usize, + pub column: usize, +} diff --git a/src/api/tokens/mod.rs b/src/api/tokens/mod.rs new file mode 100644 index 0000000..f8253d5 --- /dev/null +++ b/src/api/tokens/mod.rs @@ -0,0 +1,38 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +/*! +Tokens, received from lexer output. +*/ + +/// Token value. +/// +/// The lifetime of this token refers to original source string which +/// should be kept alive as long as this token. +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum TokenRef<'a> { + BlockStart, + Name(&'a str), + Text(&'a str), +} + +impl<'a> From> for Token { + /// Get owned value for this token. + fn from<'r>(other: TokenRef<'r>) -> Self { + match other { + TokenRef::BlockStart => Token::BlockStart, + TokenRef::Name(v) => Token::Name(v.into()), + TokenRef::Text(v) => Token::Text(v.into()), + } + } +} + +/// Owned token value. +#[derive(Debug, Clone)] +pub enum Token { + BlockStart, + Name(String), + Text(String), +} diff --git a/src/lib.rs b/src/lib.rs index 23b87dc..e3f2a0b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,3 +61,4 @@ #[macro_use] pub mod error; +pub mod api; diff --git a/tests/lexer/mod.rs b/tests/lexer/mod.rs new file mode 100644 index 0000000..f78e8a5 --- /dev/null +++ b/tests/lexer/mod.rs @@ -0,0 +1,23 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +use twig::api::lexer::{ Lexer, Tokens, Options }; +use twig::api::tokens::TokenRef; + +#[test] +fn name_label_for_tag() { + let template = "{% § %}"; + let lexer = Lexer::new(Options::default(), Vec::new()); + let mut s = lexer.tokens(&template); + + expect(&mut s, TokenRef::BlockStart); + expect(&mut s, TokenRef::Name("§")); +} + +fn expect<'i, 'c>(stream: &mut Tokens<'i, 'c>, token_value: TokenRef<'c>) { + if let Err(e) = stream.expect(token_value) { + panic!("Received error {:?}", e); + } +} diff --git a/tests/lexer_tests.rs b/tests/lexer_tests.rs new file mode 100644 index 0000000..533214f --- /dev/null +++ b/tests/lexer_tests.rs @@ -0,0 +1,8 @@ +// This file is part of rust-web/twig +// +// For the copyright and license information, please view the LICENSE +// file that was distributed with this source code. + +extern crate twig; + +mod lexer;