b3b00 · b3b00 · Sep 22, 2017 · Sep 18, 2017 · Sep 21, 2017 · Sep 22, 2017
diff --git a/ParserExample/ParserExample.csproj b/ParserExample/ParserExample.csproj
@@ -13,6 +13,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\expressionParser\expressionParser.csproj" />
     <ProjectReference Include="..\jsonparser\jsonparser.csproj" />
     <ProjectReference Include="..\sly\sly.csproj" />
   </ItemGroup>

diff --git a/ParserExample/Program.cs b/ParserExample/Program.cs
@@ -1,44 +1,39 @@
-using sly.parser;
-using jsonparser;
-using sly.lexer;
+using sly.lexer;
 using sly.parser.generator;
-using System.Linq;
 using System.Collections.Generic;
-using System.IO;
+using expressionparser;
 using System;
-using jsonparser.JsonModel;
-using sly.parser.syntax;
+using System.Linq;
+using System.Reflection;
 
 namespace ParserExample
 {
 
     public enum TokenType
     {
+        [Lexeme("a")]
         a = 1,
+        [Lexeme("b")]
         b = 2,
+        [Lexeme("c")]
         c = 3,
+        [Lexeme("z")]
         z = 26,
+        [Lexeme("r")]
         r = 21,
+        [Lexeme("[ \\t]+",true)]
         WS = 100,
+        [Lexeme("[\\r\\n]+",true,true)]
         EOL = 101
     }
 
-class Program
-    {
 
-
-        public static Lexer<TokenType> BuildLexer()
-        {
-            Lexer<TokenType> lexer = new Lexer<TokenType>();
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.WS, "[ \\t]+", true));
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.EOL, "[\\n\\r]+", true, true));
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.a, "a"));
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.b, "b"));
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.c, "c"));
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.z, "z"));
-            lexer.AddDefinition(new TokenDefinition<TokenType>(TokenType.r, "r"));
-            return lexer;
-        }
+
+
+
+
+    class Program
+    {
 
         [Production("R : A b c ")]
         [Production("R : Rec b c ")]
@@ -82,16 +77,26 @@ public static object Rec(List<object> args)
 
         static void Main(string[] args)
         {
-
 
-            RuleParser<JsonToken> ruleparser = new RuleParser<JsonToken>();
-            ParserBuilder builder = new ParserBuilder();
+            ILexer<ExpressionToken> lexer = LexerBuilder.BuildLexer<ExpressionToken>();
+
+            string source = "1 + 2\n*3";
+
+
 
-            Parser<EbnfToken,GrammarNode<JsonToken>> yacc = builder.BuildParser<EbnfToken,GrammarNode<JsonToken>>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule");
 
-            var r = yacc.Parse("test : CROG INT CROD");
+
+            List<Token<ExpressionToken>> toks = lexer.Tokenize(source).ToList();
             ;
 
+            //RuleParser<JsonToken> ruleparser = new RuleParser<JsonToken>();
+            //ParserBuilder builder = new ParserBuilder();
+
+            //Parser<EbnfToken,GrammarNode<JsonToken>> yacc = builder.BuildParser<EbnfToken,GrammarNode<JsonToken>>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule");
+
+            //var r = yacc.Parse("test : CROG INT CROD");
+            //;
+
         }
     }
 }
diff --git a/ParserTests/EBNFTests.cs b/ParserTests/EBNFTests.cs
@@ -18,12 +18,19 @@ public class EBNFTests
 
         public enum TokenType
         {
+            [Lexeme("a")]
             a = 1,
+            [Lexeme("b")]
             b = 2,
+            [Lexeme("c")]
             c = 3,
+            [Lexeme("e")]
             e = 4,
+            [Lexeme("f")]
             f = 5,
+            [Lexeme("[ \\t]+",true)]
             WS = 100,
+            [Lexeme("\\n\\r]+", true, true)]
             EOL = 101
         }
 
@@ -110,20 +117,20 @@ public EBNFTests()
         private Parser<TokenType, string> BuildParser()
         {
             EBNFTests parserInstance = new EBNFTests();
-            ParserBuilder builder = new ParserBuilder();
+            ParserBuilder<TokenType, string> builder = new ParserBuilder<TokenType, string>();
 
-            Parser = builder.BuildParser<TokenType, string>(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "R");
+            Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "R");
             return Parser;
         }
 
 
         private Parser<JsonToken, JSon> BuildEbnfJsonParser()
         {
             EbnfJsonParser parserInstance = new EbnfJsonParser();
-            ParserBuilder builder = new ParserBuilder();
+            ParserBuilder<JsonToken, JSon> builder = new ParserBuilder<JsonToken, JSon>();
 
             JsonParser =
-                builder.BuildParser<JsonToken, JSon>(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
+                builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root");
             return JsonParser;
         }
 

diff --git a/ParserTests/ErrorTests.cs b/ParserTests/ErrorTests.cs
@@ -16,8 +16,8 @@ public class ErrorTests
         public void TestJsonSyntaxError()
         {
             JSONParser jsonParser = new JSONParser();
-            ParserBuilder builder = new ParserBuilder();
-            Parser<JsonToken,JSon> parser = builder.BuildParser<JsonToken,JSon>(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root");
+            ParserBuilder<JsonToken, JSon> builder = new ParserBuilder<JsonToken, JSon>();
+            Parser<JsonToken,JSon> parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root");
 
 
             string source = @"{
@@ -42,8 +42,8 @@ public void TestJsonSyntaxError()
         public void TestExpressionSyntaxError()
         {
             ExpressionParser exprParser = new ExpressionParser();
-            ParserBuilder builder = new ParserBuilder();
-            Parser<ExpressionToken,int> Parser = builder.BuildParser<ExpressionToken,int>(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression");
+            ParserBuilder<ExpressionToken, int> builder = new ParserBuilder<ExpressionToken, int>();
+            Parser<ExpressionToken,int> Parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression");
 
             ParseResult<ExpressionToken,int> r = Parser.Parse(" 2 + 3 + + 2");
             Assert.True(r.IsError);
@@ -63,8 +63,8 @@ public void TestLexicalError()
         {
             ExpressionParser exprParser = new ExpressionParser();
 
-            ParserBuilder builder = new ParserBuilder();
-            Parser<ExpressionToken,int> Parser = builder.BuildParser<ExpressionToken,int>(exprParser, ParserType.LL_RECURSIVE_DESCENT, "root");
+            ParserBuilder<ExpressionToken, int> builder = new ParserBuilder<ExpressionToken, int>();
+            Parser<ExpressionToken,int> Parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "root");
             ParseResult<ExpressionToken,int> r = Parser.Parse("2 @ 2");
             Assert.True(r.IsError);
             Assert.NotNull(r.Errors);

diff --git a/ParserTests/ExpressionTests.cs b/ParserTests/ExpressionTests.cs
@@ -17,8 +17,8 @@ public class ExpressionTests
         public ExpressionTests()
         {
             ExpressionParser parserInstance = new ExpressionParser();
-            ParserBuilder builder = new ParserBuilder();
-            Parser = builder.BuildParser<ExpressionToken,int>(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression");
+            ParserBuilder<ExpressionToken, int> builder = new ParserBuilder<ExpressionToken, int>();
+            Parser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression");
         }
 
 

diff --git a/ParserTests/JsonTests.cs b/ParserTests/JsonTests.cs
@@ -18,8 +18,8 @@ public class JsonTests
         public JsonTests()
         {
             JSONParser jsonParser = new JSONParser();
-            ParserBuilder builder = new ParserBuilder();
-            Parser = builder.BuildParser<JsonToken,JSon>(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root");
+            ParserBuilder<JsonToken, JSon> builder = new ParserBuilder<JsonToken, JSon>();
+            Parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root");
         }
 
 

diff --git a/ParserTests/LexerTests.cs b/ParserTests/LexerTests.cs
@@ -17,16 +17,16 @@ public class LexerTests
         private ILexer<JsonToken> GetJsonLexer()
         {
             JSONParser jsonParser = new JSONParser();
-            ParserBuilder builder = new ParserBuilder();
-            Parser<JsonToken,JSon> parser = builder.BuildParser<JsonToken,JSon>(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root");
+            ParserBuilder<JsonToken, JSon> builder = new ParserBuilder<JsonToken, JSon>();
+            Parser<JsonToken,JSon> parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root");
             return parser.Lexer;
         }
 
         private ILexer<ExpressionToken> GetExpressionLexer()
         {
             ExpressionParser exprParser = new ExpressionParser();
-            ParserBuilder builder = new ParserBuilder();
-            Parser<ExpressionToken,int> parser = builder.BuildParser<ExpressionToken,int>(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression");
+            ParserBuilder<ExpressionToken, int> builder = new ParserBuilder<ExpressionToken, int>();
+            Parser<ExpressionToken,int> parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression");
             return parser.Lexer;
         }
 
@@ -64,6 +64,24 @@ public void TestSingleLineJsonLexing()
         public void TestSingleLineExpressionLexing()
         {
             ILexer<ExpressionToken> lexer = GetExpressionLexer();
+            string expr = "1 + 2 * 3";
+            List<Token<ExpressionToken>> tokens = lexer.Tokenize(expr).ToList<Token<ExpressionToken>>();
+            Assert.Equal(6, tokens.Count);
+            List<ExpressionToken> expectedTokensID = new List<ExpressionToken>()
+            {
+                ExpressionToken.INT, ExpressionToken.PLUS,ExpressionToken.INT,
+                ExpressionToken.TIMES, ExpressionToken.INT
+            };
+            List<ExpressionToken> tokensID = tokens.Take(5).Select((Token<ExpressionToken> tok) => tok.TokenID).ToList<ExpressionToken>();
+            Assert.Equal(expectedTokensID, tokensID);
+
+            List<int> expectedColumnPositions = new List<int>()
+            {                
+                1,3,5,7,9
+            };
+
+            List<int> columnPositions = tokens.Take(5).Select((Token<ExpressionToken> tok) => tok.Position.Column).ToList<int>();
+            Assert.Equal(expectedColumnPositions, columnPositions);
         }
 
         [Fact]
@@ -107,6 +125,32 @@ public void TestMultiLineJsonLexing()
         public void TestMultiLineExpressionLexing()
         {
             ILexer<ExpressionToken> lexer = GetExpressionLexer();
+            string expr = "1 + 2 \n* 3";
+            List<Token<ExpressionToken>> tokens = lexer.Tokenize(expr).ToList<Token<ExpressionToken>>();
+            Assert.Equal(6, tokens.Count);
+            List<ExpressionToken> expectedTokensID = new List<ExpressionToken>()
+            {
+                ExpressionToken.INT, ExpressionToken.PLUS,ExpressionToken.INT,
+                ExpressionToken.TIMES, ExpressionToken.INT
+            };
+            List<ExpressionToken> tokensID = tokens.Take(5).Select((Token<ExpressionToken> tok) => tok.TokenID).ToList<ExpressionToken>();
+            Assert.Equal(expectedTokensID, tokensID);
+
+            List<int> expectedColumnPositions = new List<int>()
+            {   
+                1,3,5,1,3
+            };
+
+            List<int> columnPositions = tokens.Take(5).Select((Token<ExpressionToken> tok) => tok.Position.Column).ToList<int>();
+            Assert.Equal(expectedColumnPositions, columnPositions);
+
+            List<int> expectedLinePositions = new List<int>()
+            {
+                1,3,5,1,3
+            };
+
+            List<int> linePositions = tokens.Take(5).Select((Token<ExpressionToken> tok) => tok.Position.Line).ToList<int>();
+            Assert.Equal(expectedLinePositions, columnPositions);
         }
     }
 }
diff --git a/ParserTests/VisitorTests.cs b/ParserTests/VisitorTests.cs
@@ -9,10 +9,15 @@ namespace ParserTests
 {
     public enum TokenType
     {
+        [Lexeme("a")]
         a = 1,
+        [Lexeme("b")]
         b = 2,
+        [Lexeme("c")]
         c = 3,
+        [Lexeme("[ \\t]+", true)]
         WS = 100,
+        [Lexeme("[\\n\\r]+", true, true)]
         EOL = 101
     }
 
@@ -61,8 +66,8 @@ public SyntaxNode<TokenType> node(string name, params ISyntaxNode<TokenType>[] l
         public void testVisitor()
         {
             VisitorTests visitorInstance = new VisitorTests();
-            ParserBuilder builder = new ParserBuilder();
-            Parser<TokenType,string> parser = builder.BuildParser<TokenType,string>(visitorInstance, ParserType.LL_RECURSIVE_DESCENT, "R");
+            ParserBuilder<TokenType, string> builder = new ParserBuilder<TokenType, string>();
+            Parser<TokenType,string> parser = builder.BuildParser(visitorInstance, ParserType.LL_RECURSIVE_DESCENT, "R");
             SyntaxTreeVisitor<TokenType,string> visitor = parser.Visitor;
 
             // build a syntax tree