From 88108d13ae7011d54ce8aec7031c36a1175b49ce Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Thu, 24 Oct 2024 09:54:05 -0600 Subject: [PATCH 1/6] Migrate to ANTLR v4 (WIP), #977 --- .build/dependencies.props | 3 +- src/Lucene.Net.Expressions/JS/Javascript.g4 | 249 ++ .../JS/JavascriptCompiler.cs | 622 +++-- .../JS/JavascriptLexer.cs | 2177 ----------------- .../JS/JavascriptParser.cs | 1886 -------------- .../Lucene.Net.Expressions.csproj | 14 +- 6 files changed, 640 insertions(+), 4311 deletions(-) create mode 100644 src/Lucene.Net.Expressions/JS/Javascript.g4 delete mode 100644 src/Lucene.Net.Expressions/JS/JavascriptLexer.cs delete mode 100644 src/Lucene.Net.Expressions/JS/JavascriptParser.cs diff --git a/.build/dependencies.props b/.build/dependencies.props index c2c3ce6abb..5c1124752d 100644 --- a/.build/dependencies.props +++ b/.build/dependencies.props @@ -20,7 +20,8 @@ --> - 3.5.1 + 4.13.1 + 12.8.0 @@ -68,4 +66,12 @@ + + + true + false + Lucene.Net.Expressions.JS + + + From 630942c1d0c433a4201d5786d78d83d4ccb7f959 Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Sat, 26 Oct 2024 09:02:08 -0600 Subject: [PATCH 2/6] Fix bug where logical_and and logical_or rules were always emitting IL --- .../JS/JavascriptCompiler.cs | 108 ++++++++++-------- 1 file changed, 61 insertions(+), 47 deletions(-) diff --git a/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs b/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs index 47e018ddbd..ca96a93225 100644 --- a/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs +++ b/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs @@ -1,19 +1,19 @@ using Antlr4.Runtime; using Antlr4.Runtime.Tree; +using J2N; using J2N.Text; using Lucene.Net.Queries.Function; using Lucene.Net.Support; using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Linq; using System.Reflection; using System.Reflection.Emit; -using JCG = J2N.Collections.Generic; -using J2N; using System.Text; -using System.Diagnostics.CodeAnalysis; +using JCG = J2N.Collections.Generic; namespace Lucene.Net.Expressions.JS { @@ -548,70 +548,84 @@ public override void EnterBitwise_or(JavascriptParser.Bitwise_orContext context) public override void EnterLogical_and(JavascriptParser.Logical_andContext context) { - // Evaluate the first operand and check if it is false - EnterBitwise_or(context.bitwise_or(0)); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); - - // Iterate over the remaining operands - for (int i = 2; i < context.children.Count; i += 2) + if (context.children.Count == 1) + { + EnterBitwise_or(context.bitwise_or(0)); + } + else { - if (context.children[i] is not JavascriptParser.Bitwise_orContext bitwiseOrContext) + // Evaluate the first operand and check if it is false + EnterBitwise_or(context.bitwise_or(0)); + _compiler.gen.Emit(OpCodes.Ldc_I4_0); + _compiler.gen.Emit(OpCodes.Ceq); + + // Iterate over the remaining operands + for (int i = 2; i < context.children.Count; i += 2) { - throw new InvalidOperationException("Unexpected child of logical_and"); + if (context.children[i] is not JavascriptParser.Bitwise_orContext bitwiseOrContext) + { + throw new InvalidOperationException("Unexpected child of logical_and"); + } + + // Evaluate the next operand and check if it is false + EnterBitwise_or(bitwiseOrContext); + _compiler.gen.Emit(OpCodes.Ldc_I4_0); + _compiler.gen.Emit(OpCodes.Ceq); + + // Combine the results using OR + _compiler.gen.Emit(OpCodes.Or); } - // Evaluate the next operand and check if it is false - EnterBitwise_or(bitwiseOrContext); + // Check if the combined result is false _compiler.gen.Emit(OpCodes.Ldc_I4_0); _compiler.gen.Emit(OpCodes.Ceq); - // Combine the results using OR - _compiler.gen.Emit(OpCodes.Or); + // Convert the result to a double + _compiler.gen.Emit(OpCodes.Conv_R8); } - - // Check if the combined result is false - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); - - // Convert the result to a double - _compiler.gen.Emit(OpCodes.Conv_R8); } public override void EnterLogical_or(JavascriptParser.Logical_orContext context) { - // Evaluate the first operand and check if it is true - EnterLogical_and(context.logical_and(0)); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); - _compiler.gen.Emit(OpCodes.Ldc_I4_1); - _compiler.gen.Emit(OpCodes.Xor); - - // Iterate over the remaining operands - for (int i = 2; i < context.children.Count; i += 2) + if (context.children.Count == 1) { - if (context.children[i] is not JavascriptParser.Logical_andContext logicalAndContext) - { - throw new InvalidOperationException("Unexpected child of logical_or"); - } - - // Evaluate the next operand and check if it is true - EnterLogical_and(logicalAndContext); + EnterLogical_and(context.logical_and(0)); + } + else + { + // Evaluate the first operand and check if it is true + EnterLogical_and(context.logical_and(0)); _compiler.gen.Emit(OpCodes.Ldc_I4_0); _compiler.gen.Emit(OpCodes.Ceq); _compiler.gen.Emit(OpCodes.Ldc_I4_1); _compiler.gen.Emit(OpCodes.Xor); - // Combine the results using OR - _compiler.gen.Emit(OpCodes.Or); - } + // Iterate over the remaining operands + for (int i = 2; i < context.children.Count; i += 2) + { + if (context.children[i] is not JavascriptParser.Logical_andContext logicalAndContext) + { + throw new InvalidOperationException("Unexpected child of logical_or"); + } + + // Evaluate the next operand and check if it is true + EnterLogical_and(logicalAndContext); + _compiler.gen.Emit(OpCodes.Ldc_I4_0); + _compiler.gen.Emit(OpCodes.Ceq); + _compiler.gen.Emit(OpCodes.Ldc_I4_1); + _compiler.gen.Emit(OpCodes.Xor); - // Check if the combined result is true - _compiler.gen.Emit(OpCodes.Ldc_I4_1); - _compiler.gen.Emit(OpCodes.Ceq); + // Combine the results using OR + _compiler.gen.Emit(OpCodes.Or); + } - // Convert the result to a double - _compiler.gen.Emit(OpCodes.Conv_R8); + // Check if the combined result is true + _compiler.gen.Emit(OpCodes.Ldc_I4_1); + _compiler.gen.Emit(OpCodes.Ceq); + + // Convert the result to a double + _compiler.gen.Emit(OpCodes.Conv_R8); + } } public override void EnterConditional(JavascriptParser.ConditionalContext context) From cda4c6b0e6e37346c4a0d8ef17030e36d19ab9d5 Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Sun, 27 Oct 2024 13:28:11 -0600 Subject: [PATCH 3/6] Fix failing unit tests with ANTLRv4 port, except Api style ones --- .../JS/JavascriptCompiler.cs | 321 +++++++++++------- .../JS/TestJavascriptOperations.cs | 7 +- 2 files changed, 206 insertions(+), 122 deletions(-) diff --git a/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs b/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs index ca96a93225..2a57c02af2 100644 --- a/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs +++ b/src/Lucene.Net.Expressions/JS/JavascriptCompiler.cs @@ -225,11 +225,11 @@ private void RecursiveCompile(JavascriptParser.ExpressionContext context) // be equivalent to what RecursiveCompile was doing previously private class JavascriptListener : JavascriptBaseListener { - private readonly JavascriptCompiler _compiler; + private readonly JavascriptCompiler compiler; public JavascriptListener(JavascriptCompiler compiler) { - _compiler = compiler; + this.compiler = compiler; } public override void EnterExpression(JavascriptParser.ExpressionContext context) @@ -244,7 +244,7 @@ public override void EnterCall(JavascriptParser.CallContext context) // LUCENENET: logic changed to get arguments from parse context var arguments = context.arguments().conditional(); int argumentCount = arguments.Length; - if (!_compiler.functions.TryGetValue(call, out MethodInfo method) || method is null) + if (!compiler.functions.TryGetValue(call, out MethodInfo method) || method is null) { throw new ArgumentException("Unrecognized method call (" + call + ")."); } @@ -258,7 +258,7 @@ public override void EnterCall(JavascriptParser.CallContext context) { EnterConditional(arguments[argument]); } - _compiler.gen.Emit(OpCodes.Call, method); + compiler.Emit(OpCodes.Call, method); } public override void EnterPrimary(JavascriptParser.PrimaryContext context) @@ -267,19 +267,19 @@ public override void EnterPrimary(JavascriptParser.PrimaryContext context) { string text = namespaceId.GetText(); - if (!_compiler.externalsMap.TryGetValue(text, out int index)) + if (!compiler.externalsMap.TryGetValue(text, out int index)) { - _compiler.externalsMap[text] = index = _compiler.externalsMap.Count; + compiler.externalsMap[text] = index = compiler.externalsMap.Count; } - _compiler.gen.Emit(OpCodes.Nop); + compiler.Emit(OpCodes.Nop); - _compiler.gen.Emit(OpCodes.Ldarg_2); - _compiler.gen.Emit(OpCodes.Ldc_I4, index); + compiler.Emit(OpCodes.Ldarg_2); + compiler.Emit(OpCodes.Ldc_I4, index); - _compiler.gen.Emit(OpCodes.Ldelem_Ref); - _compiler.gen.Emit(OpCodes.Ldarg_1); - _compiler.gen.Emit(OpCodes.Callvirt, DOUBLE_VAL_METHOD); + compiler.Emit(OpCodes.Ldelem_Ref); + compiler.Emit(OpCodes.Ldarg_1); + compiler.Emit(OpCodes.Callvirt, DOUBLE_VAL_METHOD); } else if (context.numeric() is { } numeric) { @@ -291,7 +291,7 @@ public override void EnterPrimary(JavascriptParser.PrimaryContext context) } else { - throw new InvalidOperationException("Unknown primary alternative"); + throw new ParseException("Unknown primary alternative", context.Start.StartIndex); } } @@ -301,34 +301,15 @@ public override void EnterNumeric(JavascriptParser.NumericContext context) if (context.HEX() is not null) { - _compiler.PushInt64(Convert.ToInt64(text, 16)); + compiler.PushInt64(Convert.ToInt64(text, 16)); } else if (context.OCTAL() is not null) { - _compiler.PushInt64(Convert.ToInt64(text, 8)); + compiler.PushInt64(Convert.ToInt64(text, 8)); } else { - // decimal - //.NET Port. This is a bit hack-y but was needed since .NET can't perform bitwise ops on longs & doubles - var bitwiseOps = new[]{ ">>", "<<", "&", "~", "|", "^" }; - - if (bitwiseOps.Any(s => text.Contains(s))) - { - if (int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int val)) - { - _compiler.gen.Emit(OpCodes.Ldc_I4, val); - } - else - { - _compiler.gen.Emit(OpCodes.Ldc_I8, long.Parse(text, CultureInfo.InvariantCulture)); - _compiler.gen.Emit(OpCodes.Conv_Ovf_U4_Un); - } - } - else - { - _compiler.gen.Emit(OpCodes.Ldc_R8, double.Parse(text, CultureInfo.InvariantCulture)); - } + compiler.Emit(OpCodes.Ldc_R8, double.Parse(text, CultureInfo.InvariantCulture)); } } @@ -344,7 +325,7 @@ public override void EnterPostfix(JavascriptParser.PostfixContext context) } else { - throw new InvalidOperationException("Unknown postfix alternative"); + throw new ParseException("Unknown postfix alternative", context.Start.StartIndex); } } @@ -356,10 +337,8 @@ public override void EnterUnary(JavascriptParser.UnaryContext context) } else if (context.AT_ADD() is not null) { - // LUCENENET-specific: it appears that 4.8 had a bug where - // it would push an "add" opcode here, but that is not correct - // for the unary + operator. - throw new NotImplementedException("Unary + is not supported"); + EnterUnary(context.unary()); + compiler.Emit(OpCodes.Conv_R8); } else if (context.unary_operator() is { } unaryOperator) { @@ -367,27 +346,28 @@ public override void EnterUnary(JavascriptParser.UnaryContext context) if (unaryOperator.AT_SUBTRACT() is not null) { - _compiler.gen.Emit(OpCodes.Neg); + compiler.Emit(OpCodes.Neg); } else if (unaryOperator.AT_BIT_NOT() is not null) { - _compiler.gen.Emit(OpCodes.Not); - _compiler.gen.Emit(OpCodes.Conv_R8); + compiler.Emit(OpCodes.Conv_I8); // cast to long (truncate) + compiler.Emit(OpCodes.Not); + compiler.Emit(OpCodes.Conv_R8); } else if (unaryOperator.AT_BOOL_NOT() is not null) { - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); - _compiler.gen.Emit(OpCodes.Conv_R8); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Conv_R8); } else { - throw new InvalidOperationException("Unknown unary_operator alternative"); + throw new ParseException("Unknown unary_operator alternative", context.Start.StartIndex); } } else { - throw new InvalidOperationException("Unknown unary alternative"); + throw new ParseException("Unknown unary alternative", context.Start.StartIndex); } } @@ -397,15 +377,15 @@ public override void EnterAdditive(JavascriptParser.AdditiveContext context) { if (terminalNode.Symbol.Type == JavascriptParser.AT_ADD) { - _compiler.PushOpWithConvert(OpCodes.Add); + compiler.PushOpWithConvert(OpCodes.Add); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_SUBTRACT) { - _compiler.PushOpWithConvert(OpCodes.Sub); + compiler.PushOpWithConvert(OpCodes.Sub); } else { - throw new InvalidOperationException("Unknown additive token"); + throw new ParseException("Unknown additive token", context.Start.StartIndex); } }); } @@ -416,42 +396,50 @@ public override void EnterMultiplicative(JavascriptParser.MultiplicativeContext { if (terminalNode.Symbol.Type == JavascriptParser.AT_MULTIPLY) { - _compiler.PushOpWithConvert(OpCodes.Mul); + compiler.PushOpWithConvert(OpCodes.Mul); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_DIVIDE) { - _compiler.PushOpWithConvert(OpCodes.Div); + compiler.PushOpWithConvert(OpCodes.Div); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_MODULO) { - _compiler.PushOpWithConvert(OpCodes.Rem); + compiler.PushOpWithConvert(OpCodes.Rem); } else { - throw new InvalidOperationException("Unknown multiplicative token"); + throw new ParseException("Unknown multiplicative token", context.Start.StartIndex); } }); } public override void EnterShift(JavascriptParser.ShiftContext context) { - CompileBinary(context, context.additive, EnterAdditive, terminalNode => + CompileBinary(context, context.additive, additiveContext => + { + EnterAdditive(additiveContext); + + if (context.children.Count > 1) // if we have a shift token + { + compiler.Emit(OpCodes.Conv_I8); // cast to long (truncate) + } + }, terminalNode => { if (terminalNode.Symbol.Type == JavascriptParser.AT_BIT_SHL) { - _compiler.PushOpWithConvert(OpCodes.Shl); + compiler.PushOpWithConvert(OpCodes.Shl); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_BIT_SHR) { - _compiler.PushOpWithConvert(OpCodes.Shr); + compiler.PushOpWithConvert(OpCodes.Shr); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_BIT_SHU) { - _compiler.PushOpWithConvert(OpCodes.Shr_Un); + compiler.PushOpWithConvert(OpCodes.Shr_Un); } else { - throw new InvalidOperationException("Unknown shift token"); + throw new ParseException("Unknown shift token", context.Start.StartIndex); } }); } @@ -461,23 +449,23 @@ public override void EnterRelational(JavascriptParser.RelationalContext context) CompileBinary(context, context.shift, EnterShift, terminalNode => { if (terminalNode.Symbol.Type == JavascriptParser.AT_COMP_LT) { - _compiler.PushOpWithConvert(OpCodes.Clt); + compiler.PushOpWithConvert(OpCodes.Clt); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_COMP_GT) { - _compiler.PushOpWithConvert(OpCodes.Cgt); + compiler.PushOpWithConvert(OpCodes.Cgt); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_COMP_LTE) { - _compiler.PushCondEq(OpCodes.Cgt); + compiler.PushCondEq(OpCodes.Cgt); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_COMP_GTE) { - _compiler.PushCondEq(OpCodes.Clt); + compiler.PushCondEq(OpCodes.Clt); } else { - throw new InvalidOperationException("Unknown relational token"); + throw new ParseException("Unknown relational token", context.Start.StartIndex); } }); } @@ -488,60 +476,84 @@ public override void EnterEquality(JavascriptParser.EqualityContext context) { if (terminalNode.Symbol.Type == JavascriptParser.AT_COMP_EQ) { - _compiler.PushOpWithConvert(OpCodes.Ceq); + compiler.PushOpWithConvert(OpCodes.Ceq); } else if (terminalNode.Symbol.Type == JavascriptParser.AT_COMP_NEQ) { - _compiler.PushCondEq(OpCodes.Ceq); + compiler.PushCondEq(OpCodes.Ceq); } else { - throw new InvalidOperationException("Unknown equality token"); + throw new ParseException("Unknown equality token", context.Start.StartIndex); } }); } public override void EnterBitwise_and(JavascriptParser.Bitwise_andContext context) { - CompileBinary(context, context.equality, EnterEquality, terminalNode => + CompileBinary(context, context.equality, equalityContext => + { + EnterEquality(equalityContext); + + if (context.children.Count > 1) // if we have a bitwise token + { + compiler.Emit(OpCodes.Conv_I8); // cast to long (truncate) + } + }, terminalNode => { if (terminalNode.Symbol.Type == JavascriptParser.AT_BIT_AND) { - _compiler.PushOpWithConvert(OpCodes.And); + compiler.PushOpWithConvert(OpCodes.And); } else { - throw new InvalidOperationException("Unknown bitwise_and token"); + throw new ParseException("Unknown bitwise_and token", context.Start.StartIndex); } }); } public override void EnterBitwise_xor(JavascriptParser.Bitwise_xorContext context) { - CompileBinary(context, context.bitwise_and, EnterBitwise_and, terminalNode => + CompileBinary(context, context.bitwise_and, andContext => + { + EnterBitwise_and(andContext); + + if (context.children.Count > 1) // if we have a bitwise token + { + compiler.Emit(OpCodes.Conv_I8); // cast to long (truncate) + } + }, terminalNode => { if (terminalNode.Symbol.Type == JavascriptParser.AT_BIT_XOR) { - _compiler.PushOpWithConvert(OpCodes.Xor); + compiler.PushOpWithConvert(OpCodes.Xor); } else { - throw new InvalidOperationException("Unknown bitwise_xor token"); + throw new ParseException("Unknown bitwise_xor token", context.Start.StartIndex); } }); } public override void EnterBitwise_or(JavascriptParser.Bitwise_orContext context) { - CompileBinary(context, context.bitwise_xor, EnterBitwise_xor, terminalNode => + CompileBinary(context, context.bitwise_xor, xorContext => + { + EnterBitwise_xor(xorContext); + + if (context.children.Count > 1) // if we have a bitwise token + { + compiler.Emit(OpCodes.Conv_I8); // cast to long (truncate) + } + }, terminalNode => { if (terminalNode.Symbol.Type == JavascriptParser.AT_BIT_OR) { - _compiler.PushOpWithConvert(OpCodes.Or); + compiler.PushOpWithConvert(OpCodes.Or); } else { - throw new InvalidOperationException("Unknown bitwise_or token"); + throw new ParseException("Unknown bitwise_or token", context.Start.StartIndex); } }); } @@ -556,32 +568,32 @@ public override void EnterLogical_and(JavascriptParser.Logical_andContext contex { // Evaluate the first operand and check if it is false EnterBitwise_or(context.bitwise_or(0)); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Ceq); // Iterate over the remaining operands for (int i = 2; i < context.children.Count; i += 2) { if (context.children[i] is not JavascriptParser.Bitwise_orContext bitwiseOrContext) { - throw new InvalidOperationException("Unexpected child of logical_and"); + throw new ParseException("Unexpected child of logical_and", context.Start.StartIndex); } // Evaluate the next operand and check if it is false EnterBitwise_or(bitwiseOrContext); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Ceq); // Combine the results using OR - _compiler.gen.Emit(OpCodes.Or); + compiler.Emit(OpCodes.Or); } // Check if the combined result is false - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Ceq); // Convert the result to a double - _compiler.gen.Emit(OpCodes.Conv_R8); + compiler.Emit(OpCodes.Conv_R8); } } @@ -595,36 +607,36 @@ public override void EnterLogical_or(JavascriptParser.Logical_orContext context) { // Evaluate the first operand and check if it is true EnterLogical_and(context.logical_and(0)); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); - _compiler.gen.Emit(OpCodes.Ldc_I4_1); - _compiler.gen.Emit(OpCodes.Xor); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Ldc_I4_1); + compiler.Emit(OpCodes.Xor); // Iterate over the remaining operands for (int i = 2; i < context.children.Count; i += 2) { if (context.children[i] is not JavascriptParser.Logical_andContext logicalAndContext) { - throw new InvalidOperationException("Unexpected child of logical_or"); + throw new ParseException("Unexpected child of logical_or", context.Start.StartIndex); } // Evaluate the next operand and check if it is true EnterLogical_and(logicalAndContext); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Ceq); - _compiler.gen.Emit(OpCodes.Ldc_I4_1); - _compiler.gen.Emit(OpCodes.Xor); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Ldc_I4_1); + compiler.Emit(OpCodes.Xor); // Combine the results using OR - _compiler.gen.Emit(OpCodes.Or); + compiler.Emit(OpCodes.Or); } // Check if the combined result is true - _compiler.gen.Emit(OpCodes.Ldc_I4_1); - _compiler.gen.Emit(OpCodes.Ceq); + compiler.Emit(OpCodes.Ldc_I4_1); + compiler.Emit(OpCodes.Ceq); // Convert the result to a double - _compiler.gen.Emit(OpCodes.Conv_R8); + compiler.Emit(OpCodes.Conv_R8); } } @@ -636,24 +648,24 @@ public override void EnterConditional(JavascriptParser.ConditionalContext contex } else { - Label condFalse = _compiler.gen.DefineLabel(); - Label condEnd = _compiler.gen.DefineLabel(); + Label condFalse = compiler.gen.DefineLabel(); + Label condEnd = compiler.gen.DefineLabel(); // Evaluate the condition EnterLogical_or(context.logical_or()); - _compiler.gen.Emit(OpCodes.Ldc_I4_0); - _compiler.gen.Emit(OpCodes.Beq, condFalse); + compiler.Emit(OpCodes.Ldc_I4_0); + compiler.Emit(OpCodes.Beq, condFalse); // Evaluate the true branch EnterConditional(context.conditional(0)); - _compiler.gen.Emit(OpCodes.Br_S, condEnd); + compiler.Emit(OpCodes.Br_S, condEnd); // Evaluate the false branch - _compiler.gen.MarkLabel(condFalse); + compiler.gen.MarkLabel(condFalse); EnterConditional(context.conditional(1)); // Mark the end of the conditional - _compiler.gen.MarkLabel(condEnd); + compiler.gen.MarkLabel(condEnd); } } @@ -677,7 +689,7 @@ private static void CompileBinary( } else { - throw new InvalidOperationException("Unexpected child"); + throw new ParseException("Unexpected child", context.Start.StartIndex); } } } @@ -685,16 +697,16 @@ private static void CompileBinary( private void PushCondEq(OpCode opCode) { - gen.Emit(opCode); - gen.Emit(OpCodes.Ldc_I4_1); - gen.Emit(OpCodes.Xor); - gen.Emit(OpCodes.Conv_R8); + Emit(opCode); + Emit(OpCodes.Ldc_I4_1); + Emit(OpCodes.Xor); + Emit(OpCodes.Conv_R8); } private void PushOpWithConvert(OpCode opCode) { - gen.Emit(opCode); - gen.Emit(OpCodes.Conv_R8); + Emit(opCode); + Emit(OpCodes.Conv_R8); } /// @@ -702,13 +714,44 @@ private void PushOpWithConvert(OpCode opCode) /// private void PushInt64(long i) { - gen.Emit(OpCodes.Ldc_I8,i); + Emit(OpCodes.Ldc_I8, i); if (!sourceText.Contains("<<")) { - gen.Emit(OpCodes.Conv_R8); + Emit(OpCodes.Conv_R8); } } + // LUCENENET-specific - wrapping Emit methods which is helpful for debugging + private void Emit(OpCode opcode) + { + // Console.WriteLine(opcode); + gen.Emit(opcode); + } + + private void Emit(OpCode opcode, Label label) + { + // Console.WriteLine(opcode + " " + label); + gen.Emit(opcode, label); + } + + private void Emit(OpCode opcode, double arg) + { + // Console.WriteLine(opcode + " " + arg); + gen.Emit(opcode, arg); + } + + private void Emit(OpCode opcode, long arg) + { + // Console.WriteLine(opcode + " " + arg); + gen.Emit(opcode, arg); + } + + private void Emit(OpCode opcode, MethodInfo arg) + { + // Console.WriteLine(opcode + " " + arg); + gen.Emit(opcode, arg); + } + private void EndCompile() { gen.Emit(OpCodes.Ret); @@ -721,17 +764,59 @@ private JavascriptParser.ExpressionContext GetAntlrComputedExpressionTree() JavascriptLexer lexer = new JavascriptLexer(input); CommonTokenStream tokens = new CommonTokenStream(lexer); JavascriptParser parser = new JavascriptParser(tokens); + try { + parser.ErrorHandler = new ParseExceptionErrorStrategy(); // LUCENENET-specific return parser.expression(); } catch (RecognitionException re) { throw new ArgumentException(re.Message, re); } - // LUCENENET: Antlr 3.5.1 doesn't ever wrap ParseException, so the other catch block - // for RuntimeException that wraps ParseException would have - // been completely unnecesary in Java and is also unnecessary here. + } + + // LUCENENET-specific: Throw a ParseException in the event of parsing errors + private class ParseExceptionErrorStrategy : DefaultErrorStrategy + { + public override void Recover(Parser recognizer, RecognitionException e) + { + int errorOffset = -1; + + for (ParserRuleContext parserRuleContext = recognizer.Context; parserRuleContext != null; parserRuleContext = (ParserRuleContext)parserRuleContext.Parent) + { + if (errorOffset < 0) + { + errorOffset = parserRuleContext.Start.StartIndex; + } + + parserRuleContext.exception = e; + } + + throw new ParseException(e.Message, errorOffset); + } + + public override IToken RecoverInline(Parser recognizer) + { + InputMismatchException cause = new InputMismatchException(recognizer); + int errorOffset = -1; + + for (ParserRuleContext parserRuleContext = recognizer.Context; parserRuleContext != null; parserRuleContext = (ParserRuleContext)parserRuleContext.Parent) + { + if (errorOffset < 0) + { + errorOffset = parserRuleContext.Start.StartIndex; + } + + parserRuleContext.exception = cause; + } + + throw new ParseException(cause.Message, errorOffset); + } + + public override void Sync(Parser recognizer) + { + } } /// The default set of functions available to expressions. diff --git a/src/Lucene.Net.Tests.Expressions/JS/TestJavascriptOperations.cs b/src/Lucene.Net.Tests.Expressions/JS/TestJavascriptOperations.cs index 2bcea6cc3f..595580a195 100644 --- a/src/Lucene.Net.Tests.Expressions/JS/TestJavascriptOperations.cs +++ b/src/Lucene.Net.Tests.Expressions/JS/TestJavascriptOperations.cs @@ -269,7 +269,7 @@ public virtual void TestBitShiftLeft() AssertEvaluatesTo("4195 << 6", 268480); AssertEvaluatesTo("4195 << 70", 268480); AssertEvaluatesTo("-4195 << 70", -268480); - AssertEvaluatesTo("-15 << 62", 1073741824); + AssertEvaluatesTo("-15 << 62", 4611686018427387904L); } [Test] @@ -295,13 +295,12 @@ public virtual void TestBitShiftRightUnsigned() AssertEvaluatesTo("2 >>> 1", 1); AssertEvaluatesTo("-1 >>> 37", 134217727); AssertEvaluatesTo("-2 >>> 62", 3); - //.NET Port. CLR returns different values for unsigned shift ops - AssertEvaluatesTo("-5 >>> 33", 2147483645); + AssertEvaluatesTo("-5 >>> 33", 2147483647); AssertEvaluatesTo("536960 >>> 7", 4195); AssertEvaluatesTo("16780 >>> 66", 4195); AssertEvaluatesTo("268480 >>> 6", 4195); AssertEvaluatesTo("268480 >>> 70", 4195); - AssertEvaluatesTo("-268480 >>> 102", 67104669); + AssertEvaluatesTo("-268480 >>> 102", 67108863); AssertEvaluatesTo("2147483648 >>> 1", 1073741824); } From 905caafbb2365e45318629e83068c9b15a1a37d4 Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Sun, 27 Oct 2024 14:22:16 -0600 Subject: [PATCH 4/6] Exclude generated code from ApiScanTestBase tests --- .../Support/ApiScanTestBase.cs | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/Lucene.Net.TestFramework/Support/ApiScanTestBase.cs b/src/Lucene.Net.TestFramework/Support/ApiScanTestBase.cs index bb5ac1fd75..99b0d359a6 100644 --- a/src/Lucene.Net.TestFramework/Support/ApiScanTestBase.cs +++ b/src/Lucene.Net.TestFramework/Support/ApiScanTestBase.cs @@ -1,5 +1,6 @@ using Lucene.Net.Support; using System; +using System.CodeDom.Compiler; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -407,7 +408,7 @@ private static IEnumerable GetInvalidPrivateFields(Assembly assembly, st { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -443,7 +444,7 @@ private static IEnumerable GetInvalidProtectedFields(Assembly assembly) { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -489,7 +490,7 @@ private static IEnumerable GetInvalidPublicFields(Assembly assembly, str { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass && !t.HasAttribute(inherit: false)); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -535,7 +536,7 @@ private static IEnumerable GetInvalidMethodParameterNames(Assembly assem { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -567,7 +568,7 @@ private static IEnumerable GetInvalidInterfaceNames(Assembly assembly) { var result = new List(); - var interfaces = assembly.GetTypes().Where(t => t.IsInterface); + var interfaces = GetTypesToTest(assembly).Where(t => t.IsInterface); foreach (var i in interfaces) { @@ -584,7 +585,7 @@ private static IEnumerable GetInvalidClassNames(Assembly assembly) { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -611,7 +612,7 @@ private static IEnumerable GetPropertiesWithNoGetter(Assembly assembly) { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -633,7 +634,7 @@ private static IEnumerable GetPropertiesThatReturnArray(Assembly assembl { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -667,7 +668,7 @@ private static IEnumerable GetProtectedFieldsContainingComparer(Assembly { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -699,7 +700,7 @@ private static IEnumerable GetMembersContainingComparer(Assembly assembl { var result = new List(); - var types = assembly.GetTypes(); + var types = GetTypesToTest(assembly); foreach (var t in types) { @@ -737,7 +738,7 @@ private static IEnumerable GetMembersNamedSize(Assembly assembly) { var result = new List(); - var types = assembly.GetTypes(); + var types = GetTypesToTest(assembly); foreach (var t in types) { @@ -772,7 +773,7 @@ private static IEnumerable GetMembersContainingNonNetNumeric(Assembly as { var result = new List(); - var types = assembly.GetTypes(); + var types = GetTypesToTest(assembly); foreach (var t in types) { @@ -816,7 +817,7 @@ private static IEnumerable GetTypesContainingNonNetNumeric(Assembly asse { var result = new List(); - var types = assembly.GetTypes(); + var types = GetTypesToTest(assembly); foreach (var t in types) { @@ -838,7 +839,7 @@ private static IEnumerable GetMethodsThatReturnWritableArray(Assembly as { var result = new List(); - var classes = assembly.GetTypes().Where(t => t.IsClass); + var classes = GetTypesToTest(assembly).Where(t => t.IsClass); foreach (var c in classes) { @@ -892,7 +893,7 @@ private static IEnumerable GetPublicNullableEnumMembers(Assembly assembl { var result = new List(); - var types = assembly.GetTypes(); + var types = GetTypesToTest(assembly); foreach (var t in types) { @@ -1004,7 +1005,7 @@ private static IEnumerable GetMembersAcceptingOrReturningType(Type lookF { var result = new List(); - var types = assembly.GetTypes(); + var types = GetTypesToTest(assembly); foreach (var t in types) { @@ -1105,5 +1106,10 @@ private static IEnumerable GetMembersAcceptingOrReturningType(Type lookF return result.ToArray(); } + + private static IEnumerable GetTypesToTest(Assembly assembly) => + assembly.GetTypes() + .Where(t => !t.HasAttribute(inherit: false) + && !t.HasAttribute(inherit: false)); } } From 7570539136e22753b0ce9400aac2a9ec1a40d140 Mon Sep 17 00:00:00 2001 From: Paul Irwin Date: Sun, 27 Oct 2024 14:26:17 -0600 Subject: [PATCH 5/6] Make Antlr4BuildTasks a build-time-only dependency, and remove old ANTLRv3 transitive deps --- src/Lucene.Net.Expressions/Lucene.Net.Expressions.csproj | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Lucene.Net.Expressions/Lucene.Net.Expressions.csproj b/src/Lucene.Net.Expressions/Lucene.Net.Expressions.csproj index 3e5c00c530..8c00990971 100644 --- a/src/Lucene.Net.Expressions/Lucene.Net.Expressions.csproj +++ b/src/Lucene.Net.Expressions/Lucene.Net.Expressions.csproj @@ -44,10 +44,7 @@ - - - - + From 407293b0d894f4ce1273456d5344bde8f0f5a69e Mon Sep 17 00:00:00 2001 From: Shad Storhaug Date: Mon, 11 Nov 2024 16:23:38 +0700 Subject: [PATCH 6/6] LICENSE.txt: Added BSD license for src/Lucene.Net.Expressions/JS/Javascript.g4. Normalized dividers between different license attributions. --- LICENSE.txt | 64 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index ed5e1374b6..fe107390ae 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -201,7 +201,7 @@ See the License for the specific language governing permissions and limitations under the License. - +======== Some code in src/Lucene.Net/Util/UnicodeUtil.cs was derived from unicode conversion examples available at http://www.unicode.org/Public/PROGRAMS/CVTUTF. @@ -230,6 +230,7 @@ Here is the copyright from those sources: * remains attached. */ +======== Some code in src/Lucene.Net/Util/ArrayUtil.cs was derived from Python 2.4.2 sources available at http://www.python.org. @@ -238,6 +239,7 @@ Full license is here: http://www.python.org/download/releases/2.4.2/license/ +======== Some code in src/Lucene.Net/Util/UnicodeUtil.cs was derived from Python 3.1.2 sources available at @@ -245,6 +247,7 @@ http://www.python.org. Full license is here: http://www.python.org/download/releases/3.1.2/license/ +======== Some code in src/Lucene.Net/Util/Automaton was derived from Brics automaton sources available at @@ -277,6 +280,7 @@ www.brics.dk/automaton/. Here is the copyright from those sources: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +======== The levenshtein automata tables in src/Lucene.Net/Util/Automaton were automatically generated with the moman/finenight FSA package. @@ -305,6 +309,7 @@ Here is the copyright for those sources: # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. +======== Some code in src/Lucene.Net/Util/UnicodeUtil.cs was derived from ICU (http://www.icu-project.org) @@ -338,7 +343,8 @@ The full license is available here: * dealings in this Software without prior written authorization of the * copyright holder. */ - + +======== The following license applies to the Snowball stemmers: @@ -369,6 +375,7 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF this SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +======== The following license applies to src/Lucene.Net.Analysis.Common/Analysis/En/KStemmer.cs: @@ -540,7 +547,7 @@ Office or Microsoft Dynamics). Some code in src/Lucene.Net.Analysis.Stempel/Egothor.Stemmer and src/Lucene.Net.Tests.Analysis.Stempel/Egothor.Stemmer falls -under the following license +under the following license: Egothor Software License version 1.00 Copyright (C) 1997-2004 Leo Galambos. @@ -643,7 +650,7 @@ falls under the following license: // Written by David Megginson // NO WARRANTY! This class is in the public domain. ---- +======== The code in .build/psake/ falls under the following license: @@ -668,7 +675,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- +======== src/Lucene.Net/Support/Compatibility/NullableAttributes.cs src/Lucene.Net/Support/DateTimeOffsetUtil.cs @@ -696,7 +703,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- +======== Some code in src/Lucene.Net/Support/ConcurrentHashSet.cs falls under the following license: @@ -722,7 +729,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- +======== Some code in src/Lucene.Net.TestFramework/Support/Util/DefaultNamespaceTypeWrapper.cs @@ -749,3 +756,46 @@ falls under the following license: // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. + +======== + +The code in src/Lucene.Net.Expressions/JS/Javascript.g4 falls under the following license: + +Copyrights 2008-2009 Xebic Reasearch BV. All rights reserved.. +Original work by Patrick Hulsmeijer. + +This ANTLR 3 LL(*) grammar is based on Ecma-262 3rd edition (JavaScript 1.5, JScript 5.5). +The annotations refer to the "A Grammar Summary" section (e.g. A.1 Lexical Grammar) +and the numbers in parenthesis to the paragraph numbers (e.g. (7.8) ). +This document is best viewed with ANTLRWorks (www.antlr.org). + +Software License Agreement (BSD License) + +Copyright (c) 2008-2010, Xebic Research B.V. +All rights reserved. + +Redistribution and use of this software in source and binary forms, with or without modification, are +permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above + copyright notice, this list of conditions and the + following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the + following disclaimer in the documentation and/or other + materials provided with the distribution. + + * Neither the name of Xebic Research B.V. nor the names of its + contributors may be used to endorse or promote products + derived from this software without specific prior + written permission of Xebic Research B.V. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR +TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file