Added Pattern Fragment matching. Removed pattern merging. Added opti…

…on to ignore whitespace in EBNF conversion.
synfron · Apr 21, 2022 · 48134ed · 48134ed
1 parent 956dfc0
commit 48134ed
Show file tree

Hide file tree

Showing 37 changed files with 3,437 additions and 2,273 deletions.
diff --git a/docs/Matcher/Language-Matcher-Schema.md b/docs/Matcher/Language-Matcher-Schema.md
diff --git a/docs/Matcher/LanguageMatcherDefinition.schema.json b/docs/Matcher/LanguageMatcherDefinition.schema.json
diff --git a/src/Matcher.CLI/Matcher.CLI.csproj b/src/Matcher.CLI/Matcher.CLI.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <AssemblyName>matchercli</AssemblyName>
     <RootNamespace>Synfron.Staxe.Matcher.CLI</RootNamespace>
     <Authors>Daquanne Dwight</Authors>

diff --git a/src/Matcher.Interop.Ebnf/CharacterClassMatchEngine.cs b/src/Matcher.Interop.Ebnf/CharacterClassMatchEngine.cs
diff --git a/src/Matcher.Interop.Ebnf/EbnfConverter.cs b/src/Matcher.Interop.Ebnf/EbnfConverter.cs
@@ -65,7 +65,10 @@ public LanguageMatcher Convert(string name, string ebnfText)
 
 			AddRules((FragmentMatchData)matcherResult.MatchData);
 
-			_languageMatcher.IndexingMode = IndexingMode.Eager;
+			if (IgnoreWhitespace)
+            {
+				_languageMatcher.IndexingMode = IndexingMode.Eager;
+            }
 			_languageMatcher.StartingFragment = _languageMatcher.Fragments.FirstOrDefault();
 
 			return _languageMatcher;

diff --git a/src/Matcher.Interop.Ebnf/EbnfMatchEngine.cs b/src/Matcher.Interop.Ebnf/EbnfMatchEngine.cs
diff --git a/src/Matcher.Interop.Ebnf/Matcher.Interop.Ebnf.csproj b/src/Matcher.Interop.Ebnf/Matcher.Interop.Ebnf.csproj
@@ -4,7 +4,7 @@
     <TargetFramework>netstandard2.0</TargetFramework>
     <RootNamespace>Synfron.Staxe.Matcher.Interop.Bnf</RootNamespace>
     <AssemblyName>Synfron.Staxe.Matcher.Interop.Bnf</AssemblyName>
-    <Version>2.0.0</Version>
+    <Version>2.0.1</Version>
     <Authors>Daquanne Dwight</Authors>
     <Company>Synfron</Company>
     <Copyright>© Daquanne Dwight</Copyright>

diff --git a/src/Matcher.Interop.Model/DefinitionConverter.cs b/src/Matcher.Interop.Model/DefinitionConverter.cs
@@ -50,7 +50,7 @@ private static IList<string> ProcessMatcherActions(LanguageMatcherDefinition lan
                 Dictionary<string, CreateVariableMatcherAction> createVariableMap = new Dictionary<string, CreateVariableMatcherAction>();
                 foreach (MatcherActionDefinition actionDefinition in languageMatcherModel.Actions)
                 {
-                    if (actionDefinition.ActionType == MatcherActionType.CreateVariable)
+                    if (actionDefinition.Action == MatcherActionType.CreateVariable)
                     {
                         CreateVariableMatcherAction action = new CreateVariableMatcherAction
                         {
@@ -68,7 +68,7 @@ private static IList<string> ProcessMatcherActions(LanguageMatcherDefinition lan
                 foreach (MatcherActionDefinition actionDefinition in languageMatcherModel.Actions)
                 {
                     MatcherAction action = null;
-                    switch (actionDefinition.ActionType)
+                    switch (actionDefinition.Action)
                     {
                         case MatcherActionType.Assert:
                             {
@@ -97,7 +97,7 @@ private static IList<string> ProcessMatcherActions(LanguageMatcherDefinition lan
                         case MatcherActionType.CreateVariable:
                             break;
                         default:
-                            throw new InvalidOperationException($"Action type {actionDefinition.ActionType} is not supported");
+                            throw new InvalidOperationException($"Action type {actionDefinition.Action} is not supported");
                     }
                 }
             }
@@ -117,7 +117,6 @@ private static List<PatternMatcher> ProcessPatternMatchers(LanguageMatcherDefini
                     {
                         PatternMatcher patternMatcher = isEager ? PatternReader.Parse(patternIndex + 1, model.Name, model.Pattern) : PatternReader.LazyParse(patternIndex + 1, model.Name, model.Pattern);
                         patternMatcher.IsNoise = model.IsNoise;
-                        patternMatcher.Mergable = model.Mergable;
                         patternMatcherMap.Add(patternMatcher.Name, patternMatcher);
                         if (!model.IsAuxiliary)
                         {
@@ -245,7 +244,6 @@ PatternMatcherDefinition ConvertPattern(PatternMatcher patternMatcher, bool isAu
 					{
                         IsAuxiliary = isAuxilary,
 						IsNoise = patternMatcher.IsNoise,
-						Mergable = patternMatcher.Mergable,
 						Name = patternMatcher.Name,
 						Pattern = patternMatcher is GroupPatternMatcher groupPatternMatcher ? groupPatternMatcher.ToString(true) : patternMatcher.ToString()
 					};
@@ -258,7 +256,7 @@ MatcherActionDefinition ConvertMatcherAction(MatcherAction matcherAction)
                     case UpdateVariableMatcherAction updateVariableMatcherAction:
                         return new MatcherActionDefinition
                         {
-                            ActionType = updateVariableMatcherAction.ActionType,
+                            Action = updateVariableMatcherAction.Action,
                             Name = updateVariableMatcherAction.Name,
                             Change = updateVariableMatcherAction.Change,
                             FirstVariableName = languageMatcher.Blobs[updateVariableMatcherAction.TargetBlobId],
@@ -267,7 +265,7 @@ MatcherActionDefinition ConvertMatcherAction(MatcherAction matcherAction)
                     case CreateVariableMatcherAction createVariableMatcherAction:
                         return new MatcherActionDefinition
                         {
-                            ActionType = createVariableMatcherAction.ActionType,
+                            Action = createVariableMatcherAction.Action,
                             Name = createVariableMatcherAction.Name,
                             Source = createVariableMatcherAction.Source,
                             FirstVariableName = languageMatcher.Blobs[createVariableMatcherAction.BlobId],
@@ -276,7 +274,7 @@ MatcherActionDefinition ConvertMatcherAction(MatcherAction matcherAction)
                     case AssertMatcherAction assertMatcherAction:
                         return new MatcherActionDefinition
                         {
-                            ActionType = assertMatcherAction.ActionType,
+                            Action = assertMatcherAction.Action,
                             Name = assertMatcherAction.Name,
                             Assert = assertMatcherAction.Assert,
                             FirstVariableName = languageMatcher.Blobs[assertMatcherAction.FirstBlobId],

diff --git a/src/Matcher.Interop.Model/Matcher.Interop.Model.csproj b/src/Matcher.Interop.Model/Matcher.Interop.Model.csproj
@@ -4,7 +4,7 @@
     <TargetFramework>netstandard2.0</TargetFramework>
     <RootNamespace>Synfron.Staxe.Matcher.Interop.Model</RootNamespace>
     <AssemblyName>Synfron.Staxe.Matcher.Interop.Model</AssemblyName>
-    <Version>2.0.0</Version>
+    <Version>3.0.0</Version>
     <Authors>Daquanne Dwight</Authors>
     <Company>Synfron</Company>
     <Copyright>© Daquanne Dwight</Copyright>

diff --git a/src/Matcher.Interop.Model/MatcherActionDefinition.cs b/src/Matcher.Interop.Model/MatcherActionDefinition.cs
@@ -1,12 +1,13 @@
 using Synfron.Staxe.Matcher.Input.Actions;
+using System;
 
 namespace Synfron.Staxe.Matcher.Interop.Model
 {
 	public class MatcherActionDefinition
 	{
 		public string Name { get; set; }
 
-		public MatcherActionType ActionType { get; set; }
+		public MatcherActionType Action { get; set; }
 
 		public string FirstVariableName { get; set; }
 
@@ -18,6 +19,6 @@ public class MatcherActionDefinition
 
 		public AssertType Assert { get; set; }
 
-		public object Value { get; set; }
+		public IConvertible Value { get; set; }
 	}
 }
diff --git a/src/Matcher.Interop.Model/PatternMatcherDefinition.cs b/src/Matcher.Interop.Model/PatternMatcherDefinition.cs
@@ -8,8 +8,6 @@ public class PatternMatcherDefinition
 
 		public bool IsNoise { get; set; }
 
-		public bool Mergable { get; set; }
-
 		public string Fragment { get; set; }
 
 		public bool IsAuxiliary { get; set; }

diff --git a/src/Matcher/AbstractLanguageMatchEngine.cs b/src/Matcher/AbstractLanguageMatchEngine.cs
@@ -13,7 +13,6 @@ public abstract class AbstractLanguageMatchEngine : ILanguageMatchEngine
 		protected ref struct State
 		{
 			public int CurrentIndex;
-			public int MaxIndex;
 			public List<StringMatchData> DistinctStringMatches;
 			public int Id;
 			public StringBuilder MatchLogBuilder;

diff --git a/src/Matcher/Data/Extensions.cs b/src/Matcher/Data/Extensions.cs
@@ -4,7 +4,7 @@
 
 namespace Synfron.Staxe.Matcher.Data
 {
-    internal static class Extensions
+    public static class Extensions
     {
 
         public static int GetEndIndex(this IMatchData matchData)

diff --git a/src/Matcher/Data/StringMatchData.cs b/src/Matcher/Data/StringMatchData.cs
@@ -12,10 +12,6 @@ public class StringMatchData : IMatchData
 
 		public string Name { get; set; }
 
-		public bool IsNoise { get; set; }
-
-		public bool Mergable { get; set; }
-
 		public int Id { get; set; }
 
 		public string ToXml()

diff --git a/src/Matcher/Input/Actions/AssertMatcherAction.cs b/src/Matcher/Input/Actions/AssertMatcherAction.cs
@@ -15,7 +15,7 @@ public class AssertMatcherAction : MatcherAction
 
         public AssertType Assert { get; set; }
 
-        public override MatcherActionType ActionType => MatcherActionType.Assert;
+        public override MatcherActionType Action => MatcherActionType.Assert;
 
         public override bool Perform(Span<BlobData> blobDatas, IList<IMatchData> matchDatas)
         {
@@ -27,26 +27,26 @@ public override bool Perform(Span<BlobData> blobDatas, IList<IMatchData> matchDa
                     return !Equals(blobDatas[FirstBlobId].Value?.ToString(), blobDatas[SecondBlobId].Value?.ToString());
                 case AssertType.GreaterThan:
                     {
-                        return double.TryParse(blobDatas[FirstBlobId].Value.ToString(), out double num1) 
-                            && double.TryParse(blobDatas[SecondBlobId].Value.ToString(), out double num2) ?
+                        return double.TryParse(blobDatas[FirstBlobId].Value?.ToString(), out double num1) 
+                            && double.TryParse(blobDatas[SecondBlobId].Value?.ToString(), out double num2) ?
                             num1 > num2 : false;
                     }
                 case AssertType.GreaterThanOrEquals:
                     {
-                        return double.TryParse(blobDatas[FirstBlobId].Value.ToString(), out double num1)
-                            && double.TryParse(blobDatas[SecondBlobId].Value.ToString(), out double num2) ?
+                        return double.TryParse(blobDatas[FirstBlobId].Value?.ToString(), out double num1)
+                            && double.TryParse(blobDatas[SecondBlobId].Value?.ToString(), out double num2) ?
                             num1 >= num2 : false;
                     }
                 case AssertType.LessThan:
                     {
-                        return double.TryParse(blobDatas[FirstBlobId].Value.ToString(), out double num1)
-                            && double.TryParse(blobDatas[SecondBlobId].Value.ToString(), out double num2) ?
+                        return double.TryParse(blobDatas[FirstBlobId].Value?.ToString(), out double num1)
+                            && double.TryParse(blobDatas[SecondBlobId].Value?.ToString(), out double num2) ?
                             num1 < num2 : false;
                     }
                 case AssertType.LessThanOrEquals:
                     {
-                        return double.TryParse(blobDatas[FirstBlobId].Value.ToString(), out double num1)
-                            && double.TryParse(blobDatas[SecondBlobId].Value.ToString(), out double num2) ?
+                        return double.TryParse(blobDatas[FirstBlobId].Value?.ToString(), out double num1)
+                            && double.TryParse(blobDatas[SecondBlobId].Value?.ToString(), out double num2) ?
                             num1 <= num2 : false;
                     }
                 case AssertType.Contains:
@@ -71,10 +71,10 @@ internal override string Generate(MatcherEngineGenerator generator)
 				string code = $@"private bool {methodName}(Span<BlobData> blobDatas, IList<IMatchData> matchDatas)
         {{
             {(Assert == AssertType.Equals ? $@"
-            return Equals(blobDatas[{FirstBlobId}]?.Value, blobDatas[{SecondBlobId}]?.Value);
+            return Equals(blobDatas[{FirstBlobId}].Value?.ToString(), blobDatas[{SecondBlobId}].Value?.ToString());
             " : null)}
             {(Assert == AssertType.NotEquals ? $@"
-            return !Equals(blobDatas[{FirstBlobId}]?.Value, blobDatas[{SecondBlobId}]?.Value);
+            return !Equals(blobDatas[{FirstBlobId}].Value?.ToString(), blobDatas[{SecondBlobId}].Value?.ToString());
             " : null)}
             {(Assert == AssertType.GreaterThan ? $@"
             return double.TryParse(blobDatas[{FirstBlobId}].Value?.ToString(), out double num1) 

diff --git a/src/Matcher/Input/Actions/CreateVariableMatcherAction.cs b/src/Matcher/Input/Actions/CreateVariableMatcherAction.cs
@@ -1,6 +1,7 @@
 using Synfron.Staxe.Matcher.Data;
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 
 namespace Synfron.Staxe.Matcher.Input.Actions
@@ -11,9 +12,9 @@ public class CreateVariableMatcherAction : MatcherAction
 
         public VariableValueSource Source { get; set; }
 
-        public object Value { get; set; }
+        public IConvertible Value { get; set; }
 
-        public override MatcherActionType ActionType => MatcherActionType.CreateVariable;
+        public override MatcherActionType Action => MatcherActionType.CreateVariable;
 
         public override bool Perform(Span<BlobData> blobDatas, IList<IMatchData> matchDatas)
         {
@@ -25,10 +26,10 @@ public override bool Perform(Span<BlobData> blobDatas, IList<IMatchData> matchDa
                         Value = Value
                     };
                     break;
-                case VariableValueSource.PartsText:
+                case VariableValueSource.PartsXml:
                     blobDatas[BlobId] = new BlobData
                     {
-                        Value = matchDatas.GetText(true)
+                        Value = string.Join("", matchDatas.Select(matchData => matchData.ToXml()))
                     };
                     break;
                 case VariableValueSource.PartsLength:
@@ -71,13 +72,13 @@ internal override string Generate(MatcherEngineGenerator generator)
                 {(Source == VariableValueSource.Value ? $@"
                 blobDatas[{BlobId}] = new BlobData
                 {{
-                    Value = Value
+                    Value = {(Value is string str ? $"\"{str}\"" : Value)}
                 }};
                 " : null)}
-                {(Source == VariableValueSource.PartsText ? $@"
+                {(Source == VariableValueSource.PartsXml ? $@"
                 blobDatas[{BlobId}] = new BlobData
                 {{
-                    Value = GetText(matchDatas)
+                    Value = string.Join("", matchDatas.Select(matchData => matchData.ToXml()))
                 }};
                 " : null)}
                 {(Source == VariableValueSource.PartsLength ? $@"

diff --git a/src/Matcher/Input/Actions/MatcherAction.cs b/src/Matcher/Input/Actions/MatcherAction.cs
@@ -10,7 +10,7 @@ public abstract class MatcherAction
     {
         public string Name { get; set; }
 
-        public abstract MatcherActionType ActionType { get; }
+        public abstract MatcherActionType Action { get; }
 
         public abstract bool Perform(Span<BlobData> blobDatas, IList<IMatchData> matchDatas);
 

diff --git a/src/Matcher/Input/Actions/UpdateVariableMatcherAction.cs b/src/Matcher/Input/Actions/UpdateVariableMatcherAction.cs
@@ -13,7 +13,7 @@ public class UpdateVariableMatcherAction : MatcherAction
 
         public int SourceBlobId { get; set; }
 
-        public override MatcherActionType ActionType => MatcherActionType.UpdateVariable;
+        public override MatcherActionType Action => MatcherActionType.UpdateVariable;
 
         public override bool Perform(Span<BlobData> blobDatas, IList<IMatchData> matchDatas)
         {

diff --git a/src/Matcher/Input/Actions/VariableValueSource.cs b/src/Matcher/Input/Actions/VariableValueSource.cs
@@ -8,7 +8,7 @@ public enum VariableValueSource
     {
         Value,
         PartsCount,
-        PartsText,
+        PartsXml,
         PartsLength,
         StringPartsText,
         StringPartsLength

diff --git a/src/Matcher/Input/FallThroughMode.cs b/src/Matcher/Input/FallThroughMode.cs
@@ -11,4 +11,12 @@ public enum FallThroughMode
         One = 1,
         All = int.MaxValue
     }
+
+    public static class FallThroughModeExtensions
+    {
+        public static bool IsCountBased(this FallThroughMode mode)
+        {
+            return FallThroughMode.None < mode && mode < FallThroughMode.All;
+        }
+    }
 }
diff --git a/src/Matcher/Input/Patterns/PatternMatcher.cs b/src/Matcher/Input/Patterns/PatternMatcher.cs
@@ -12,8 +12,6 @@ public abstract class PatternMatcher : IMatcher
 
 		public bool IsNoise { get; set; }
 
-		public bool Mergable { get; set; }
-
 		public bool IsStandard { get; set; } = true;
 
 		public abstract (bool success, int offset) IsMatch(string text, int startOffset = 0);