Lexer skeleton

src/api/error/mod.rs

@@ -0,0 +1,8 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+mod syntax;
+
+pub use self::syntax::{ SyntaxError, SyntaxErrorCode, UnexpectedToken };

src/api/error/syntax.rs

@@ -0,0 +1,44 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+use api::Position;
+use api::tokens::{ Token };
+
+#[derive(Debug)]
+pub struct SyntaxError {
+    pub code: SyntaxErrorCode,
+    pub starts_at: Position,
+    pub ends_at: Option<Position>,
+}
+
+impl SyntaxError {
+    /// Specify the location where the error ends.
+    pub fn ends_at(mut self, pos: Position) -> SyntaxError {
+        self.ends_at = Some(pos);
+        self
+    }
+}
+
+#[derive(Debug)]
+pub enum SyntaxErrorCode {
+    ExpectedTokenButReceived { expected: Token, received: UnexpectedToken },
+}
+
+impl SyntaxErrorCode {
+    /// Specify the location where the error starts.
+    pub fn starts_at(self, pos: Position) -> SyntaxError {
+        SyntaxError {
+            code: self,
+            starts_at: pos,
+            ends_at: None,
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+pub enum UnexpectedToken {
+    Token(Token),
+    EndOfStream,
+}

src/api/lexer/error.rs

@@ -0,0 +1,28 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+use std::io;
+use api::error::SyntaxError;
+use std::result;
+
+#[derive(Debug)]
+pub enum LexingError {
+    Syntax(SyntaxError),
+    Io(io::Error)
+}
+
+impl From<io::Error> for LexingError {
+    fn from(other: io::Error) -> LexingError {
+        LexingError::Io(other)
+    }
+}
+
+impl From<SyntaxError> for LexingError {
+    fn from(other: SyntaxError) -> LexingError {
+        LexingError::Syntax(other)
+    }
+}
+
+pub type LexingResult<T> = result::Result<T, LexingError>;

src/api/lexer/lexing.rs

@@ -0,0 +1,83 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+use api::Position;
+use api::tokens::TokenRef;
+
+use api::lexer::{ Lexer, LexingResult };
+use api::error::{ SyntaxErrorCode, UnexpectedToken };
+
+/// TokenRef wrapper for `Lexer` that additionaly has position.
+#[derive(Debug)]
+pub struct ItemRef<'t> {
+    pub token: TokenRef<'t>,
+    pub position: Position,
+}
+
+/// Lexer token iterator.
+///
+/// 'i is iteration lifetime, or "one use of lexer".
+/// 't is template lifetime. It will live longer than this iteration.
+pub struct Tokens<'i, 't> {
+    /// Position of the next token to get.
+    next_pos: Position, // temporary field until I get cursor in.
+    env: &'i Lexer,
+    code: &'t str,
+}
+
+impl<'i, 't> Iterator for Tokens<'i, 't> {
+    type Item = LexingResult<ItemRef<'t>>;
+
+    fn next(&mut self) -> Option<LexingResult<ItemRef<'t>>> {
+        // Hello, my name is Lexer. Twig Lexer.
+        // I am not very complicated.
+        match self.next_pos {
+            Position { line: 1, .. } => {
+                self.next_pos.line = 2;
+                Some(Ok(ItemRef { token: TokenRef::BlockStart, position: self.next_pos }))
+            },
+            Position { line: 2, .. } => {
+                self.next_pos.line = 3;
+                Some(Ok(ItemRef { token: TokenRef::Name("§"), position: self.next_pos }))
+            },
+            _ => None
+        }
+    }
+}
+
+impl<'i, 't> Tokens<'i, 't> {
+    pub fn new<'ii, 'tt>(lexer: &'ii Lexer, code: &'tt str) -> Tokens<'ii, 'tt> {
+        Tokens {
+            next_pos: Position { line: 1, column: 1 },
+            env: lexer,
+            code: code,
+        }
+    }
+
+    pub fn expect(&mut self, expected: TokenRef<'t>) -> LexingResult<TokenRef<'t>> {
+        let maybe_item = self.next();
+        match maybe_item {
+            None => Err(
+                SyntaxErrorCode::ExpectedTokenButReceived {
+                    expected: expected.into(),
+                    received: UnexpectedToken::EndOfStream
+                }.starts_at(self.next_pos).into()
+            ),
+            Some(Ok(item)) => {
+                if item.token == expected {
+                    Ok(item.token)
+                } else {
+                    Err(
+                        SyntaxErrorCode::ExpectedTokenButReceived {
+                            expected: expected.into(),
+                            received: UnexpectedToken::Token(item.token.into())
+                        }.starts_at(item.position).into()
+                    )
+                }
+            },
+            Some(Err(error)) => Err(error),
+        }
+    }
+}

src/api/lexer/mod.rs

@@ -0,0 +1,54 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+/*!
+Twig lexer.
+
+Produces a token stream from source template.
+
+# Summary
+
+This module is capable of taking a Twig input template, for example, this one:
+
+```twig
+Hello
+{% if world %}
+    world
+{% else %}
+    {{ other }}
+{% endif %}
+```
+
+And chopping it into tokens like these:
+
+TODO: Example code for this.
+*/
+
+mod lexing;
+mod error;
+
+pub use self::lexing::{ Tokens, ItemRef };
+pub use self::error::{ LexingError, LexingResult };
+
+#[derive(Copy, Clone)]
+pub struct Options;
+
+impl Options {
+    pub fn default() -> Options { Options }
+}
+
+pub struct Lexer;
+
+impl Lexer {
+    /// Create a new lexer from options and operator list.
+    pub fn new(options: Options, operators: Vec<&'static str>) -> Lexer {
+        Lexer
+    }
+
+    /// Get a lexed stream of tokens.
+    pub fn tokens<'i, 't>(&'i self, code: &'t str) -> Tokens<'i, 't> {
+        Tokens::new(self, code)
+    }
+}

src/api/mod.rs

@@ -0,0 +1,19 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+/*!
+Twig extension writer's API.
+*/
+
+pub mod tokens;
+pub mod lexer;
+pub mod error;
+
+/// Line-column position in a file.
+#[derive(Debug, Default, Copy, Clone)]
+pub struct Position {
+    pub line: usize,
+    pub column: usize,
+}

src/api/tokens/mod.rs

@@ -0,0 +1,38 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+/*!
+Tokens, received from lexer output.
+*/
+
+/// Token value.
+///
+/// The lifetime of this token refers to original source string which
+/// should be kept alive as long as this token.
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum TokenRef<'a> {
+    BlockStart,
+    Name(&'a str),
+    Text(&'a str),
+}
+
+impl<'a> From<TokenRef<'a>> for Token {
+    /// Get owned value for this token.
+    fn from<'r>(other: TokenRef<'r>) -> Self {
+        match other {
+            TokenRef::BlockStart => Token::BlockStart,
+            TokenRef::Name(v) => Token::Name(v.into()),
+            TokenRef::Text(v) => Token::Text(v.into()),
+        }
+    }
+}
+
+/// Owned token value.
+#[derive(Debug, Clone)]
+pub enum Token {
+    BlockStart,
+    Name(String),
+    Text(String),
+}

src/lib.rs

@@ -61,3 +61,4 @@
 
 #[macro_use]
 pub mod error;
+pub mod api;

tests/lexer/mod.rs

@@ -0,0 +1,23 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+use twig::api::lexer::{ Lexer, Tokens, Options };
+use twig::api::tokens::TokenRef;
+
+#[test]
+fn name_label_for_tag() {
+    let template = "{% § %}";
+    let lexer = Lexer::new(Options::default(), Vec::new());
+    let mut s = lexer.tokens(&template);
+
+    expect(&mut s, TokenRef::BlockStart);
+    expect(&mut s, TokenRef::Name("§"));
+}
+
+fn expect<'i, 'c>(stream: &mut Tokens<'i, 'c>, token_value: TokenRef<'c>) {
+    if let Err(e) = stream.expect(token_value) {
+        panic!("Received error {:?}", e);
+    }
+}

tests/lexer_tests.rs

@@ -0,0 +1,8 @@
+// This file is part of rust-web/twig
+//
+// For the copyright and license information, please view the LICENSE
+// file that was distributed with this source code.
+
+extern crate twig;
+
+mod lexer;