From c4a73b14f38efc0529cdb07717c17486fc29b30b Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Mon, 18 Sep 2017 13:54:31 +0200 Subject: [PATCH 01/10] update readme.md --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4623f20d..57ecc891 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,9 @@ you can now use EBNF notation : * ```List``` for a repeated non terminal * ```List>``` for a repeated terminal - See (https://github.com/b3b00/csly/blob/master/jsonparser/EBNFJSONParser.cs) for a complete EBNF json parser. + See [EBNFJsonParser.cs](https://github.com/b3b00/csly/blob/master/jsonparser/EBNFJSONParser.cs) for a complete EBNF json parser. +#### under the hood meta consideration on EBNF parsers #### + +The EBNF notation has been implemented in CSLY using the BNF notation. The EBNF parser builder is built using the BNF parser builder. Incidently the EBNF parser builder is a good and complete example for BNF parser : [RuleParser.cs](https://github.com/b3b00/csly/blob/master/sly/parser/generator/RuleParser.cs) From 1d600ca7b85f8e5372d6d73c2e92fd98676c6027 Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Thu, 21 Sep 2017 13:57:55 +0200 Subject: [PATCH 02/10] lexer configuration --- ParserExample/ParserExample.csproj | 1 + ParserExample/Program.cs | 59 +++++++++++++++++------ ParserTests/EBNFTests.cs | 8 +-- ParserTests/ErrorTests.cs | 12 ++--- ParserTests/ExpressionTests.cs | 4 +- ParserTests/JsonTests.cs | 4 +- ParserTests/LexerTests.cs | 8 +-- ParserTests/VisitorTests.cs | 4 +- expressionParser/ExpressionParser.cs | 1 + expressionParser/ExpressionToken.cs | 47 ++++++++++++++---- sly.sln | 15 +++--- sly/lexer/LexemeAttribute.cs | 23 +++++++++ sly/lexer/LexerBuilder.cs | 54 +++++++++++++++++++++ sly/parser/generator/EBNFParserBuilder.cs | 17 +++---- sly/parser/generator/ParserBuilder.cs | 42 ++++++---------- sly/parser/parser/Parser.cs | 3 +- 16 files changed, 214 insertions(+), 88 deletions(-) create mode 100644 sly/lexer/LexemeAttribute.cs create mode 100644 sly/lexer/LexerBuilder.cs diff --git a/ParserExample/ParserExample.csproj b/ParserExample/ParserExample.csproj index 655c640e..10f5e0c9 100644 --- a/ParserExample/ParserExample.csproj +++ b/ParserExample/ParserExample.csproj @@ -13,6 +13,7 @@ + diff --git a/ParserExample/Program.cs b/ParserExample/Program.cs index d507b5cd..142d7bcb 100644 --- a/ParserExample/Program.cs +++ b/ParserExample/Program.cs @@ -1,20 +1,19 @@ -using sly.parser; -using jsonparser; -using sly.lexer; +using sly.lexer; using sly.parser.generator; -using System.Linq; using System.Collections.Generic; -using System.IO; +using expressionparser; using System; -using jsonparser.JsonModel; -using sly.parser.syntax; +using System.Linq; +using System.Reflection; namespace ParserExample { public enum TokenType { + [Description("one")] a = 1, + [Description("two")] b = 2, c = 3, z = 26, @@ -23,7 +22,26 @@ public enum TokenType EOL = 101 } -class Program + + [AttributeUsage(AttributeTargets.All, AllowMultiple = false, Inherited = true)] + public class DescriptionAttribute : Attribute + { + + public string Description { get; set; } + + public bool IsSkippable { get; set; } + + public bool IsEnding { get; set; } + + public DescriptionAttribute(string description) + { + Description = description; + } + } + + + + class Program { @@ -83,14 +101,27 @@ public static object Rec(List args) static void Main(string[] args) { - - RuleParser ruleparser = new RuleParser(); - ParserBuilder builder = new ParserBuilder(); - Parser> yacc = builder.BuildParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); + DescriptionAttribute attr = TokenType.a.GetAttributeOfType(); + string desc = attr.Description; + + ILexer lexer = LexerBuilder.BuildLexer(); + + + ExpressionParser exprParser = new ExpressionParser(); + + ParserBuilder builder = new ParserBuilder(); + var p = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression"); + var r = p.Parse("1", "expression"); + + + //RuleParser ruleparser = new RuleParser(); + //ParserBuilder builder = new ParserBuilder(); + + //Parser> yacc = builder.BuildParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); - var r = yacc.Parse("test : CROG INT CROD"); - ; + //var r = yacc.Parse("test : CROG INT CROD"); + //; } } diff --git a/ParserTests/EBNFTests.cs b/ParserTests/EBNFTests.cs index 46806308..52fe2af1 100644 --- a/ParserTests/EBNFTests.cs +++ b/ParserTests/EBNFTests.cs @@ -110,9 +110,9 @@ public EBNFTests() private Parser BuildParser() { EBNFTests parserInstance = new EBNFTests(); - ParserBuilder builder = new ParserBuilder(); + ParserBuilder builder = new ParserBuilder(); - Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "R"); + Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "R"); return Parser; } @@ -120,10 +120,10 @@ private Parser BuildParser() private Parser BuildEbnfJsonParser() { EbnfJsonParser parserInstance = new EbnfJsonParser(); - ParserBuilder builder = new ParserBuilder(); + ParserBuilder builder = new ParserBuilder(); JsonParser = - builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); return JsonParser; } diff --git a/ParserTests/ErrorTests.cs b/ParserTests/ErrorTests.cs index c23caad9..7b7c2290 100644 --- a/ParserTests/ErrorTests.cs +++ b/ParserTests/ErrorTests.cs @@ -16,8 +16,8 @@ public class ErrorTests public void TestJsonSyntaxError() { JSONParser jsonParser = new JSONParser(); - ParserBuilder builder = new ParserBuilder(); - Parser parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root"); + ParserBuilder builder = new ParserBuilder(); + Parser parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root"); string source = @"{ @@ -42,8 +42,8 @@ public void TestJsonSyntaxError() public void TestExpressionSyntaxError() { ExpressionParser exprParser = new ExpressionParser(); - ParserBuilder builder = new ParserBuilder(); - Parser Parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression"); + ParserBuilder builder = new ParserBuilder(); + Parser Parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression"); ParseResult r = Parser.Parse(" 2 + 3 + + 2"); Assert.True(r.IsError); @@ -63,8 +63,8 @@ public void TestLexicalError() { ExpressionParser exprParser = new ExpressionParser(); - ParserBuilder builder = new ParserBuilder(); - Parser Parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "root"); + ParserBuilder builder = new ParserBuilder(); + Parser Parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "root"); ParseResult r = Parser.Parse("2 @ 2"); Assert.True(r.IsError); Assert.NotNull(r.Errors); diff --git a/ParserTests/ExpressionTests.cs b/ParserTests/ExpressionTests.cs index 3097221e..64458203 100644 --- a/ParserTests/ExpressionTests.cs +++ b/ParserTests/ExpressionTests.cs @@ -17,8 +17,8 @@ public class ExpressionTests public ExpressionTests() { ExpressionParser parserInstance = new ExpressionParser(); - ParserBuilder builder = new ParserBuilder(); - Parser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression"); + ParserBuilder builder = new ParserBuilder(); + Parser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression"); } diff --git a/ParserTests/JsonTests.cs b/ParserTests/JsonTests.cs index c56ec58e..d0677b45 100644 --- a/ParserTests/JsonTests.cs +++ b/ParserTests/JsonTests.cs @@ -18,8 +18,8 @@ public class JsonTests public JsonTests() { JSONParser jsonParser = new JSONParser(); - ParserBuilder builder = new ParserBuilder(); - Parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root"); + ParserBuilder builder = new ParserBuilder(); + Parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root"); } diff --git a/ParserTests/LexerTests.cs b/ParserTests/LexerTests.cs index 8ba3ca53..df5e7773 100644 --- a/ParserTests/LexerTests.cs +++ b/ParserTests/LexerTests.cs @@ -17,16 +17,16 @@ public class LexerTests private ILexer GetJsonLexer() { JSONParser jsonParser = new JSONParser(); - ParserBuilder builder = new ParserBuilder(); - Parser parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root"); + ParserBuilder builder = new ParserBuilder(); + Parser parser = builder.BuildParser(jsonParser, ParserType.LL_RECURSIVE_DESCENT, "root"); return parser.Lexer; } private ILexer GetExpressionLexer() { ExpressionParser exprParser = new ExpressionParser(); - ParserBuilder builder = new ParserBuilder(); - Parser parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression"); + ParserBuilder builder = new ParserBuilder(); + Parser parser = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression"); return parser.Lexer; } diff --git a/ParserTests/VisitorTests.cs b/ParserTests/VisitorTests.cs index 9777b207..f500b837 100644 --- a/ParserTests/VisitorTests.cs +++ b/ParserTests/VisitorTests.cs @@ -61,8 +61,8 @@ public SyntaxNode node(string name, params ISyntaxNode[] l public void testVisitor() { VisitorTests visitorInstance = new VisitorTests(); - ParserBuilder builder = new ParserBuilder(); - Parser parser = builder.BuildParser(visitorInstance, ParserType.LL_RECURSIVE_DESCENT, "R"); + ParserBuilder builder = new ParserBuilder(); + Parser parser = builder.BuildParser(visitorInstance, ParserType.LL_RECURSIVE_DESCENT, "R"); SyntaxTreeVisitor visitor = parser.Visitor; // build a syntax tree diff --git a/expressionParser/ExpressionParser.cs b/expressionParser/ExpressionParser.cs index 7b5b4fb6..04b5a6f3 100644 --- a/expressionParser/ExpressionParser.cs +++ b/expressionParser/ExpressionParser.cs @@ -16,6 +16,7 @@ public class ExpressionParser [LexerConfiguration] public ILexer BuildExpressionLexer(ILexer lexer) { + lexer.AddDefinition(new TokenDefinition(ExpressionToken.DOUBLE, "[0-9]+\\.[0-9]+")); lexer.AddDefinition(new TokenDefinition(ExpressionToken.INT, "[0-9]+")); lexer.AddDefinition(new TokenDefinition(ExpressionToken.PLUS, "\\+")); diff --git a/expressionParser/ExpressionToken.cs b/expressionParser/ExpressionToken.cs index 700ed134..bf6ae96b 100644 --- a/expressionParser/ExpressionToken.cs +++ b/expressionParser/ExpressionToken.cs @@ -1,20 +1,49 @@ using System; using System.Collections.Generic; using System.Text; +using sly.lexer; namespace expressionparser { public enum ExpressionToken { - INT = 2, // integer - DOUBLE = 3, // float number - PLUS = 4, // the + operator - MINUS = 5, // the - operator - TIMES = 6, // the * operator - DIVIDE = 7, // the / operator - LPAREN = 8, // a left paranthesis ( - RPAREN = 9,// a right paranthesis ) - WS = 12, // a whitespace + // float number + [Lexeme("[0-9]+\\.[0-9]+")] + DOUBLE = 1, + + // integer + [Lexeme("[0-9]+")] + INT = 3, + + // the + operator + [Lexeme("\\+")] + PLUS = 4, + + // the - operator + [Lexeme("\\-")] + MINUS = 5, + + // the * operator + [Lexeme("\\*")] + TIMES = 6, + + // the / operator + [Lexeme("\\/")] + DIVIDE = 7, + + // a left paranthesis ( + [Lexeme("\\(")] + LPAREN = 8, + + // a right paranthesis ) + [Lexeme("\\)")] + RPAREN = 9, + + // a whitespace + [Lexeme("[ \\t]+",true)] + WS = 12, + + [Lexeme("[\\n\\r] + ", true, true)] EOL = 14 } } diff --git a/sly.sln b/sly.sln index 05371a07..db1ad4a8 100644 --- a/sly.sln +++ b/sly.sln @@ -1,14 +1,14 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 -VisualStudioVersion = 15.0.26228.4 +VisualStudioVersion = 15.0.26730.12 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "sly", "sly\sly.csproj", "{E10CD5E3-0049-45C1-BE38-5D0EF074881D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "sly", "sly\sly.csproj", "{E10CD5E3-0049-45C1-BE38-5D0EF074881D}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "jsonparser", "jsonparser\jsonparser.csproj", "{A0A0A026-C902-4646-A22E-BD5BBCD4AB6A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParserTests", "ParserTests\ParserTests.csproj", "{3BE9D05E-B17B-4A4B-9639-A48E51857A51}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ParserTests", "ParserTests\ParserTests.csproj", "{3BE9D05E-B17B-4A4B-9639-A48E51857A51}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ParserExample", "ParserExample\ParserExample.csproj", "{9CF0151C-B25A-463D-ACEA-1F6DEDCFD91E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ParserExample", "ParserExample\ParserExample.csproj", "{9CF0151C-B25A-463D-ACEA-1F6DEDCFD91E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "expressionParser", "expressionParser\expressionParser.csproj", "{21FA2DD5-A47C-457B-BFB6-F9B24205FD7E}" EndProject @@ -30,8 +30,8 @@ Global {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|Any CPU.ActiveCfg = Release|Any CPU {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|Any CPU.Build.0 = Release|Any CPU {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|x64.ActiveCfg = Release|Any CPU - {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|x86.ActiveCfg = Release|x86 - {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|x86.Build.0 = Release|x86 + {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|x86.ActiveCfg = Release|Any CPU + {E10CD5E3-0049-45C1-BE38-5D0EF074881D}.Release|x86.Build.0 = Release|Any CPU {A0A0A026-C902-4646-A22E-BD5BBCD4AB6A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {A0A0A026-C902-4646-A22E-BD5BBCD4AB6A}.Debug|Any CPU.Build.0 = Debug|Any CPU {A0A0A026-C902-4646-A22E-BD5BBCD4AB6A}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -82,6 +82,9 @@ Global GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {43254130-CF3E-480E-952F-E50CA5D2E417} + EndGlobalSection GlobalSection(Performance) = preSolution HasPerformanceSessions = true EndGlobalSection diff --git a/sly/lexer/LexemeAttribute.cs b/sly/lexer/LexemeAttribute.cs new file mode 100644 index 00000000..a8a4e42f --- /dev/null +++ b/sly/lexer/LexemeAttribute.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace sly.lexer +{ + [AttributeUsage(AttributeTargets.All, AllowMultiple = false,Inherited =true)] + public class LexemeAttribute : Attribute + { + + public string Pattern { get; set; } + + public bool IsSkippable { get; set; } + + public bool IsEnding { get; set; } + + public LexemeAttribute(string pattern, bool isSkippable = false, bool isEnding = false) { + Pattern = pattern; + IsSkippable = isSkippable; + IsEnding = isEnding; + } + } +} diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs new file mode 100644 index 00000000..f975312d --- /dev/null +++ b/sly/lexer/LexerBuilder.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; + +namespace sly.lexer +{ + + public static class EnumHelper + { + /// + /// Gets an attribute on an enum field value + /// + /// The type of the attribute you want to retrieve + /// The enum value + /// The attribute of type T that exists on the enum value + /// string desc = myEnumVariable.GetAttributeOfType().Description; + public static T GetAttributeOfType(this Enum enumVal) where T : System.Attribute + { + var type = enumVal.GetType(); + var memInfo = type.GetMember(enumVal.ToString()); + IEnumerable attributes = (IEnumerable)(memInfo[0].GetCustomAttributes(typeof(T), false)); + return (T)attributes?.ToArray()[0]; + } + } + + public class LexerBuilder + { + + public static ILexer BuildLexer() where T:struct + { + Type type = typeof(T); + TypeInfo typeInfo = type.GetTypeInfo(); + ILexer lexer = new Lexer(); + + var values = Enum.GetValues(typeof(T)); + + var fields = typeof(T).GetFields(); + foreach(Enum value in values) + { + LexemeAttribute lexem = value.GetAttributeOfType(); + if (lexem != null) + { + lexer.AddDefinition(new TokenDefinition(default(T), lexem.Pattern, lexem.IsSkippable, lexem.IsEnding)); + } + ; + } + + return lexer; + } + + } +} diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index 26ae4f2f..d707d8cb 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -20,25 +20,22 @@ namespace sly.parser.generator /// /// this class provides API to build parser /// - internal class EBNFParserBuilder : ParserBuilder + internal class EBNFParserBuilder : ParserBuilder where IN :struct { - - private Parser> GrammarParser; - public EBNFParserBuilder() { } - public virtual Parser BuildParser(object parserInstance, ParserType parserType, string rootRule) + public override Parser BuildParser(object parserInstance, ParserType parserType, string rootRule) { RuleParser ruleparser = new RuleParser(); - ParserBuilder builder = new ParserBuilder(); + ParserBuilder> builder = new ParserBuilder>(); - Parser> GrammarParser = builder.BuildParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); + Parser> grammarParser = builder.BuildParser(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); //ParserBuilder builder = new ParserBuilder(); // EBNFParserBuilder> parserGrammar = new EBNFParserBuilder>(); @@ -47,7 +44,7 @@ public virtual Parser BuildParser(object parserInstance, ParserType pars // GrammarParser = builder.BuildParser>(parserGrammar, ParserType.LL_RECURSIVE_DESCENT, "rule"); // } ParserConfiguration configuration = - ExtractEbnfParserConfiguration(parserInstance.GetType(), GrammarParser); + ExtractEbnfParserConfiguration(parserInstance.GetType(), grammarParser); ISyntaxParser syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); @@ -62,7 +59,7 @@ public virtual Parser BuildParser(object parserInstance, ParserType pars } Parser parser = new Parser(syntaxParser, visitor); parser.Configuration = configuration; - parser.Lexer = BuildLexer(parserInstance.GetType(), parserInstance); + parser.Lexer = BuildLexer(); parser.Instance = parserInstance; return parser; } @@ -84,7 +81,7 @@ public ILexer BuildEbnfLexer(ILexer lexer) - protected override ISyntaxParser BuildSyntaxParser(ParserConfiguration conf, ParserType parserType, + protected override ISyntaxParser BuildSyntaxParser(ParserConfiguration conf, ParserType parserType, string rootRule) { ISyntaxParser parser = null; diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index 238428d4..bc723374 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -16,7 +16,7 @@ namespace sly.parser.generator /// /// this class provides API to build parser /// - public class ParserBuilder + public class ParserBuilder where IN : struct { #region API @@ -30,16 +30,16 @@ public class ParserBuilder /// a ParserType enum value stating the analyser type (LR, LL ...) for now only LL recurive descent parser available /// the name of the root non terminal of the grammar /// - public virtual Parser BuildParser(object parserInstance, ParserType parserType, string rootRule) + public virtual Parser BuildParser(object parserInstance, ParserType parserType, string rootRule) { Parser parser = null; if (parserType == ParserType.LL_RECURSIVE_DESCENT) { - ParserConfiguration configuration = ExtractParserConfiguration(parserInstance.GetType()); - ISyntaxParser syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); + ParserConfiguration configuration = ExtractParserConfiguration(parserInstance.GetType()); + ISyntaxParser syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); SyntaxTreeVisitor visitor = new SyntaxTreeVisitor(configuration, parserInstance); parser = new Parser(syntaxParser, visitor); - parser.Lexer = BuildLexer(parserInstance.GetType(), parserInstance); + parser.Lexer = BuildLexer(); parser.Instance = parserInstance; parser.Configuration = configuration; } @@ -51,7 +51,7 @@ public virtual Parser BuildParser(object parserInstance, ParserT return parser; } - protected virtual ISyntaxParser BuildSyntaxParser(ParserConfiguration conf, ParserType parserType, string rootRule) + protected virtual ISyntaxParser BuildSyntaxParser(ParserConfiguration conf, ParserType parserType, string rootRule) { ISyntaxParser parser = null; switch (parserType) @@ -99,33 +99,19 @@ private Tuple ExtractNTAndRule(string ruleString) } - protected Lexer BuildLexer(Type parserClass, object parserInstance = null) + protected ILexer BuildLexer() where IN : struct { - TypeInfo typeInfo = parserClass.GetTypeInfo(); - Lexer lexer = null; - List < MethodInfo > methods = typeInfo.DeclaredMethods.ToList(); - methods = methods.Where(m => - { - List attributes = m.GetCustomAttributes().ToList(); - Attribute attr = attributes.Find(a => a.GetType() == typeof(LexerConfigurationAttribute)); - return attr != null; - }).ToList(); - if (methods.Count > 0) - { - MethodInfo lexerConfigurerMethod = methods[0]; - lexer = new Lexer(); - object res = lexerConfigurerMethod.Invoke(parserInstance, new object[] { lexer }); - } + ILexer lexer = LexerBuilder.BuildLexer(); return lexer; } - protected virtual ParserConfiguration ExtractParserConfiguration(Type parserClass) + protected virtual ParserConfiguration ExtractParserConfiguration(Type parserClass) { - ParserConfiguration conf = new ParserConfiguration(); + ParserConfiguration conf = new ParserConfiguration(); Dictionary functions = new Dictionary(); - Dictionary> nonTerminals = new Dictionary>(); + Dictionary> nonTerminals = new Dictionary>(); List methods = parserClass.GetMethods().ToList(); methods = methods.Where(m => { @@ -146,13 +132,13 @@ protected virtual ParserConfiguration ExtractParserConfiguration( - Rule r = BuildNonTerminal(ntAndRule); + Rule r = BuildNonTerminal(ntAndRule); string key = ntAndRule.Item1 + "__" + r.Key; functions[key] = m; - NonTerminal nonT = null; + NonTerminal nonT = null; if (!nonTerminals.ContainsKey(ntAndRule.Item1)) { - nonT = new NonTerminal(ntAndRule.Item1, new List>()); + nonT = new NonTerminal(ntAndRule.Item1, new List>()); } else { diff --git a/sly/parser/parser/Parser.cs b/sly/parser/parser/Parser.cs index 34cc9706..6c0b79a7 100644 --- a/sly/parser/parser/Parser.cs +++ b/sly/parser/parser/Parser.cs @@ -10,7 +10,7 @@ namespace sly.parser { public class Parser { - public Lexer Lexer { get; set; } + public ILexer Lexer { get; set; } public object Instance { get; set; } public ISyntaxParser SyntaxParser { get; set; } public SyntaxTreeVisitor Visitor { get; set; } @@ -29,6 +29,7 @@ public ParseResult Parse(string source, string startingNonTerminal = nul ParseResult result = null; try { + IList> tokens = Lexer.Tokenize(source).ToList>(); result = Parse(tokens, startingNonTerminal); } From e09cde8567004e93a004c4480eba91309b49cf4d Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 09:57:17 +0200 Subject: [PATCH 03/10] light refactoring --- sly/lexer/LexemeAttribute.cs | 8 ++++---- sly/lexer/LexerBuilder.cs | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sly/lexer/LexemeAttribute.cs b/sly/lexer/LexemeAttribute.cs index a8a4e42f..be9aa2ab 100644 --- a/sly/lexer/LexemeAttribute.cs +++ b/sly/lexer/LexemeAttribute.cs @@ -4,7 +4,7 @@ namespace sly.lexer { - [AttributeUsage(AttributeTargets.All, AllowMultiple = false,Inherited =true)] + [AttributeUsage(AttributeTargets.Field, AllowMultiple = false,Inherited =true)] public class LexemeAttribute : Attribute { @@ -12,12 +12,12 @@ public class LexemeAttribute : Attribute public bool IsSkippable { get; set; } - public bool IsEnding { get; set; } + public bool IsLineEnding { get; set; } - public LexemeAttribute(string pattern, bool isSkippable = false, bool isEnding = false) { + public LexemeAttribute(string pattern, bool isSkippable = false, bool isLineEnding = false) { Pattern = pattern; IsSkippable = isSkippable; - IsEnding = isEnding; + IsLineEnding = isLineEnding; } } } diff --git a/sly/lexer/LexerBuilder.cs b/sly/lexer/LexerBuilder.cs index f975312d..55fd7017 100644 --- a/sly/lexer/LexerBuilder.cs +++ b/sly/lexer/LexerBuilder.cs @@ -21,6 +21,7 @@ public static T GetAttributeOfType(this Enum enumVal) where T : System.Attrib var type = enumVal.GetType(); var memInfo = type.GetMember(enumVal.ToString()); IEnumerable attributes = (IEnumerable)(memInfo[0].GetCustomAttributes(typeof(T), false)); + return (T)attributes?.ToArray()[0]; } } @@ -39,10 +40,12 @@ public static ILexer BuildLexer() where T:struct var fields = typeof(T).GetFields(); foreach(Enum value in values) { + T tokenID = (T)(object)value; + LexemeAttribute lexem = value.GetAttributeOfType(); if (lexem != null) { - lexer.AddDefinition(new TokenDefinition(default(T), lexem.Pattern, lexem.IsSkippable, lexem.IsEnding)); + lexer.AddDefinition(new TokenDefinition(tokenID, lexem.Pattern, lexem.IsSkippable, lexem.IsLineEnding)); } ; } From 5d0cf5d383278320949bf983ae067d4cd81df772 Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 09:57:50 +0200 Subject: [PATCH 04/10] Unit Tests --- ParserExample/Program.cs | 13 +++---- ParserTests/EBNFTests.cs | 7 ++++ ParserTests/LexerTests.cs | 44 +++++++++++++++++++++++ ParserTests/VisitorTests.cs | 5 +++ expressionParser/ExpressionToken.cs | 2 +- jsonparser/JsonToken.cs | 24 +++++++++---- sly/parser/generator/EBNFParserBuilder.cs | 2 +- sly/parser/generator/EbnfToken.cs | 9 ++++- sly/parser/generator/ParserBuilder.cs | 4 +-- 9 files changed, 91 insertions(+), 19 deletions(-) diff --git a/ParserExample/Program.cs b/ParserExample/Program.cs index 142d7bcb..fbb47bbb 100644 --- a/ParserExample/Program.cs +++ b/ParserExample/Program.cs @@ -100,20 +100,17 @@ public static object Rec(List args) static void Main(string[] args) { + + ILexer lexer = LexerBuilder.BuildLexer(); + string source = "1 + 2\n*3"; - DescriptionAttribute attr = TokenType.a.GetAttributeOfType(); - string desc = attr.Description; - - ILexer lexer = LexerBuilder.BuildLexer(); - ExpressionParser exprParser = new ExpressionParser(); - ParserBuilder builder = new ParserBuilder(); - var p = builder.BuildParser(exprParser, ParserType.LL_RECURSIVE_DESCENT, "expression"); - var r = p.Parse("1", "expression"); + List> toks = lexer.Tokenize(source).ToList(); + ; //RuleParser ruleparser = new RuleParser(); //ParserBuilder builder = new ParserBuilder(); diff --git a/ParserTests/EBNFTests.cs b/ParserTests/EBNFTests.cs index 52fe2af1..78a31215 100644 --- a/ParserTests/EBNFTests.cs +++ b/ParserTests/EBNFTests.cs @@ -18,12 +18,19 @@ public class EBNFTests public enum TokenType { + [Lexeme("a")] a = 1, + [Lexeme("b")] b = 2, + [Lexeme("c")] c = 3, + [Lexeme("e")] e = 4, + [Lexeme("f")] f = 5, + [Lexeme("[ \\t]+",true)] WS = 100, + [Lexeme("\\n\\r]+", true, true)] EOL = 101 } diff --git a/ParserTests/LexerTests.cs b/ParserTests/LexerTests.cs index df5e7773..399a39db 100644 --- a/ParserTests/LexerTests.cs +++ b/ParserTests/LexerTests.cs @@ -64,6 +64,24 @@ public void TestSingleLineJsonLexing() public void TestSingleLineExpressionLexing() { ILexer lexer = GetExpressionLexer(); + string expr = "1 + 2 * 3"; + List> tokens = lexer.Tokenize(expr).ToList>(); + Assert.Equal(6, tokens.Count); + List expectedTokensID = new List() + { + ExpressionToken.INT, ExpressionToken.PLUS,ExpressionToken.INT, + ExpressionToken.TIMES, ExpressionToken.INT + }; + List tokensID = tokens.Take(5).Select((Token tok) => tok.TokenID).ToList(); + Assert.Equal(expectedTokensID, tokensID); + + List expectedColumnPositions = new List() + { + 1,3,5,7,9 + }; + + List columnPositions = tokens.Take(5).Select((Token tok) => tok.Position.Column).ToList(); + Assert.Equal(expectedColumnPositions, columnPositions); } [Fact] @@ -107,6 +125,32 @@ public void TestMultiLineJsonLexing() public void TestMultiLineExpressionLexing() { ILexer lexer = GetExpressionLexer(); + string expr = "1 + 2 \n* 3"; + List> tokens = lexer.Tokenize(expr).ToList>(); + Assert.Equal(6, tokens.Count); + List expectedTokensID = new List() + { + ExpressionToken.INT, ExpressionToken.PLUS,ExpressionToken.INT, + ExpressionToken.TIMES, ExpressionToken.INT + }; + List tokensID = tokens.Take(5).Select((Token tok) => tok.TokenID).ToList(); + Assert.Equal(expectedTokensID, tokensID); + + List expectedColumnPositions = new List() + { + 1,3,5,1,3 + }; + + List columnPositions = tokens.Take(5).Select((Token tok) => tok.Position.Column).ToList(); + Assert.Equal(expectedColumnPositions, columnPositions); + + List expectedLinePositions = new List() + { + 1,3,5,1,3 + }; + + List linePositions = tokens.Take(5).Select((Token tok) => tok.Position.Line).ToList(); + Assert.Equal(expectedLinePositions, columnPositions); } } } diff --git a/ParserTests/VisitorTests.cs b/ParserTests/VisitorTests.cs index f500b837..26db4238 100644 --- a/ParserTests/VisitorTests.cs +++ b/ParserTests/VisitorTests.cs @@ -9,10 +9,15 @@ namespace ParserTests { public enum TokenType { + [Lexeme("a")] a = 1, + [Lexeme("b")] b = 2, + [Lexeme("c")] c = 3, + [Lexeme("[ \\t]+", true)] WS = 100, + [Lexeme("[\\n\\r]+", true, true)] EOL = 101 } diff --git a/expressionParser/ExpressionToken.cs b/expressionParser/ExpressionToken.cs index bf6ae96b..ff3b9e03 100644 --- a/expressionParser/ExpressionToken.cs +++ b/expressionParser/ExpressionToken.cs @@ -43,7 +43,7 @@ public enum ExpressionToken [Lexeme("[ \\t]+",true)] WS = 12, - [Lexeme("[\\n\\r] + ", true, true)] + [Lexeme("[\\n\\r]+", true, true)] EOL = 14 } } diff --git a/jsonparser/JsonToken.cs b/jsonparser/JsonToken.cs index 0e4030d7..a7e31dc0 100644 --- a/jsonparser/JsonToken.cs +++ b/jsonparser/JsonToken.cs @@ -1,26 +1,38 @@ using System; using System.Collections.Generic; using System.Text; +using sly.lexer; namespace jsonparser { public enum JsonToken { + [Lexeme("(\\\")([^(\\\")]*)(\\\")")] STRING = 1, - INT = 2, - DOUBLE = 3, + [Lexeme("[0-9]+\\.[0-9]+")] + DOUBLE = 2, + [Lexeme("[0-9]+")] + INT = 3, + [Lexeme("(true|false)")] BOOLEAN = 4, + [Lexeme("{")] ACCG = 5, + [Lexeme("}")] ACCD = 6, + [Lexeme("\\[")] CROG = 7, + [Lexeme("\\]")] CROD = 8, + [Lexeme(",")] COMMA = 9, - COLON = 10, - SEMICOLON = 11, + [Lexeme(":")] + COLON = 10, + [Lexeme("[ \\t]+", true)] WS = 12, + [Lexeme("[\\n\\r]+", true,true)] EOL = 13, - NULL = 14, - QUOTE = 99 + [Lexeme("(null)")] + NULL = 14, } } diff --git a/sly/parser/generator/EBNFParserBuilder.cs b/sly/parser/generator/EBNFParserBuilder.cs index d707d8cb..9aac777d 100644 --- a/sly/parser/generator/EBNFParserBuilder.cs +++ b/sly/parser/generator/EBNFParserBuilder.cs @@ -59,7 +59,7 @@ public override Parser BuildParser(object parserInstance, ParserType par } Parser parser = new Parser(syntaxParser, visitor); parser.Configuration = configuration; - parser.Lexer = BuildLexer(); + parser.Lexer = BuildLexer(); parser.Instance = parserInstance; return parser; } diff --git a/sly/parser/generator/EbnfToken.cs b/sly/parser/generator/EbnfToken.cs index 8612f4ea..8753ad6f 100644 --- a/sly/parser/generator/EbnfToken.cs +++ b/sly/parser/generator/EbnfToken.cs @@ -1,17 +1,24 @@ using System; using System.Collections.Generic; using System.Text; +using sly.lexer; namespace sly.parser.generator { public enum EbnfToken { + + [Lexeme("[A-Za-z0-9_��������][A-Za-z0-9_��������]*") ] IDENTIFIER = 1, + [Lexeme(":")] COLON = 2, + [Lexeme("\\*")] ZEROORMORE = 3, + [Lexeme("\\+")] ONEORMORE = 4, - + [Lexeme("[ \\t]+",true)] WS = 5, + [Lexeme("\\n\\r]+",true,true)] EOL = 6 } diff --git a/sly/parser/generator/ParserBuilder.cs b/sly/parser/generator/ParserBuilder.cs index bc723374..4c626db7 100644 --- a/sly/parser/generator/ParserBuilder.cs +++ b/sly/parser/generator/ParserBuilder.cs @@ -39,7 +39,7 @@ public virtual Parser BuildParser(object parserInstance, ParserType pars ISyntaxParser syntaxParser = BuildSyntaxParser(configuration, parserType, rootRule); SyntaxTreeVisitor visitor = new SyntaxTreeVisitor(configuration, parserInstance); parser = new Parser(syntaxParser, visitor); - parser.Lexer = BuildLexer(); + parser.Lexer = BuildLexer(); parser.Instance = parserInstance; parser.Configuration = configuration; } @@ -99,7 +99,7 @@ private Tuple ExtractNTAndRule(string ruleString) } - protected ILexer BuildLexer() where IN : struct + protected ILexer BuildLexer() { ILexer lexer = LexerBuilder.BuildLexer(); return lexer; From 01d36f9ce12740c6aeb02aa7c7db336612ba1e83 Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 10:03:55 +0200 Subject: [PATCH 05/10] fix --- ParserExample/Program.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ParserExample/Program.cs b/ParserExample/Program.cs index fbb47bbb..82786e6a 100644 --- a/ParserExample/Program.cs +++ b/ParserExample/Program.cs @@ -11,14 +11,19 @@ namespace ParserExample public enum TokenType { - [Description("one")] + [Lexeme("a")] a = 1, - [Description("two")] + [Lexeme("b")] b = 2, + [Lexeme("c")] c = 3, + [Lexeme("z")] z = 26, + [Lexeme("r")] r = 21, + [Lexeme("[ \\t]+",true)] WS = 100, + [Lexeme("[\\r\\n]+",true,true)] EOL = 101 } From 1024e5ccc53826042e7f4d67a5a12fefdce7051c Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 10:05:38 +0200 Subject: [PATCH 06/10] cleaning --- ParserExample/Program.cs | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/ParserExample/Program.cs b/ParserExample/Program.cs index 82786e6a..a2c0adcd 100644 --- a/ParserExample/Program.cs +++ b/ParserExample/Program.cs @@ -28,41 +28,13 @@ public enum TokenType } - [AttributeUsage(AttributeTargets.All, AllowMultiple = false, Inherited = true)] - public class DescriptionAttribute : Attribute - { - - public string Description { get; set; } - - public bool IsSkippable { get; set; } - - public bool IsEnding { get; set; } - - public DescriptionAttribute(string description) - { - Description = description; - } - } + class Program { - - public static Lexer BuildLexer() - { - Lexer lexer = new Lexer(); - lexer.AddDefinition(new TokenDefinition(TokenType.WS, "[ \\t]+", true)); - lexer.AddDefinition(new TokenDefinition(TokenType.EOL, "[\\n\\r]+", true, true)); - lexer.AddDefinition(new TokenDefinition(TokenType.a, "a")); - lexer.AddDefinition(new TokenDefinition(TokenType.b, "b")); - lexer.AddDefinition(new TokenDefinition(TokenType.c, "c")); - lexer.AddDefinition(new TokenDefinition(TokenType.z, "z")); - lexer.AddDefinition(new TokenDefinition(TokenType.r, "r")); - return lexer; - } - [Production("R : A b c ")] [Production("R : Rec b c ")] public static object R(List args) From a6c75433b3eacec898c53322072ad9330ce37e55 Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 10:11:07 +0200 Subject: [PATCH 07/10] upgrade to 1.2.0 move lexer configuration to token enumerate : * more readable * single placed --- sly/sly.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sly/sly.csproj b/sly/sly.csproj index dcef78bf..788810ab 100644 --- a/sly/sly.csproj +++ b/sly/sly.csproj @@ -3,7 +3,7 @@ netcoreapp1.0;net45 #LY is a parser generator halfway between parser combinators and parser generator like ANTLR b3b00 - 1.1.0 + 1.2.0 https://github.com/b3b00/sly https://github.com/b3b00/sly https://github.com/b3b00/sly/blob/master/LICENSE From 1410080ca8a21fbee008b3a2f875b474b088e373 Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 14:13:43 +0200 Subject: [PATCH 08/10] update readme.md to version 1.2.0 --- README.md | 98 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 57ecc891..c8d19c49 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ Install from the NuGet gallery GUI or with the Package Manager Console using the ```Install-Package sly``` +or with dotnet core + +```dotnet add package sly``` + ## Lexer ## @@ -41,79 +45,81 @@ It could be improved in the future. ### configuration ### -To configure a lexer 2 items has to be done : +The full lexer configuration is done in an ```enum```: +The ```enum``` is listing all the possible tokens (no special constraint here except public visibility) + +Each ```enum``` value has a ```[Lexeme]``` attribute to mark it has a lexeme. The lexeme attribute takes 3 parameters: + - + ```c# + string regex + ``` : a regular expression that captures the lexeme + - ```boolean isSkippable``` (optional, default is ```false```): a boolean , true if the lexeme must be ignored ( whitespace for example) + - ```boolean isLineending``` (optionanl, default is ```false```) : true if the lexeme matches a line end (to allow line counting while lexing). -- an ```enum``` listing all the possible tokens (no special constraint here except public visibility) -- a method with special attribute ```[LexerConfigurationAttribute]``` that associates tokens from the above enum with matching regex. - -the configuration method takes a ```Lexer``` (where T is the tokens ```enum``` parameters and returns the same lexer after having added (token,regex) associations. The lexer can be used apart from the parser. It provides a method that returns an ```IEnumerable>``` (where T is the tokens ```enum```) from a ```string``` + + ```c# IList> tokens = Lexer.Tokenize(source).ToList>(); ``` -### full example, for a mathematical parser ### -#### the tokens enum #### +You can also build only a lexer using : ```c# +ILexer lexer = LexerBuilder.BuildLexer(); +var tokens = lexer.Tokenize(source).ToList(); +``` + +### full example, for a mathematical parser ### +```c# public enum ExpressionToken { + // float number + [Lexeme("[0-9]+\\.[0-9]+")] + DOUBLE = 1, - INT = 2, // integer - - DOUBLE = 3, // float number - - PLUS = 4, // the + operator + // integer + [Lexeme("[0-9]+")] + INT = 3, - MINUS = 5, // the - operator + // the + operator + [Lexeme("\\+")] + PLUS = 4, - TIMES = 6, // the * operator + // the - operator + [Lexeme("\\-")] + MINUS = 5, - DIVIDE = 7, // the / operator + // the * operator + [Lexeme("\\*")] + TIMES = 6, - LPAREN = 8, // a left paranthesis ( + // the / operator + [Lexeme("\\/")] + DIVIDE = 7, - RPAREN = 9,// a right paranthesis ) + // a left paranthesis ( + [Lexeme("\\(")] + LPAREN = 8, - WS = 12, // a whitespace + // a right paranthesis ) + [Lexeme("\\)")] + RPAREN = 9, - EOL = 14 // an end of line + // a whitespace + [Lexeme("[ \\t]+",true)] + WS = 12, + [Lexeme("[\\n\\r]+", true, true)] + EOL = 14 } ``` -#### regular expressions - -```c# - - [LexerConfiguration] - public Lexer BuildExpressionLexer(Lexer lexer = null) - { - if (lexer == null) - { - lexer = new Lexer(); - } - - lexer.AddDefinition(new TokenDefinition(ExpressionToken.DOUBLE, "[0-9]+\\.[0-9]+")); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.INT, "[0-9]+")); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.PLUS, "\\+")); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.MINUS, "\\-")); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.TIMES, "\\*")); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.DIVIDE, "\\/")); - - lexer.AddDefinition(new TokenDefinition(ExpressionToken.LPAREN, "\\(")); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.RPAREN, "\\)")); - - lexer.AddDefinition(new TokenDefinition(ExpressionToken.WS, "[ \\t]+", true)); - lexer.AddDefinition(new TokenDefinition(ExpressionToken.EOL, "[\\n\\r]+", true, true)); - return lexer; - } -``` ## Parser ## From de98d482714258d4d9eedaf3dc1f3d23e8e81444 Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 14:24:17 +0200 Subject: [PATCH 09/10] update readme.md --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c8d19c49..355ded02 100644 --- a/README.md +++ b/README.md @@ -45,15 +45,13 @@ It could be improved in the future. ### configuration ### -The full lexer configuration is done in an ```enum```: +The full lexer configuration is done in a C# ```enum```: The ```enum``` is listing all the possible tokens (no special constraint here except public visibility) Each ```enum``` value has a ```[Lexeme]``` attribute to mark it has a lexeme. The lexeme attribute takes 3 parameters: - - ```c# - string regex - ``` : a regular expression that captures the lexeme + ```string regex``` : a regular expression that captures the lexeme - ```boolean isSkippable``` (optional, default is ```false```): a boolean , true if the lexeme must be ignored ( whitespace for example) - ```boolean isLineending``` (optionanl, default is ```false```) : true if the lexeme matches a line end (to allow line counting while lexing). From c0b6059e7d1b54a51ae4764d521b09c3adf8c09c Mon Sep 17 00:00:00 2001 From: Olivier Duhart Date: Fri, 22 Sep 2017 14:24:43 +0200 Subject: [PATCH 10/10] Update README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 355ded02..785ae2f5 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,7 @@ The full lexer configuration is done in a C# ```enum```: The ```enum``` is listing all the possible tokens (no special constraint here except public visibility) Each ```enum``` value has a ```[Lexeme]``` attribute to mark it has a lexeme. The lexeme attribute takes 3 parameters: - - - ```string regex``` : a regular expression that captures the lexeme + - ```string regex``` : a regular expression that captures the lexeme - ```boolean isSkippable``` (optional, default is ```false```): a boolean , true if the lexeme must be ignored ( whitespace for example) - ```boolean isLineending``` (optionanl, default is ```false```) : true if the lexeme matches a line end (to allow line counting while lexing).