diff --git a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs index ac17788..6c7b0b4 100644 --- a/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs +++ b/src/AElf.EntityMapping.Elasticsearch/ElasticsearchResponseHelper.cs @@ -6,6 +6,21 @@ public class ElasticsearchResponseHelper { public static string GetErrorMessage(IResponse response) { - return response.ServerError == null ? "Unknown error." : response.ServerError.ToString(); + if (response.ServerError == null) + { + if (response.OriginalException == null) + { + return "Unknown error."; + } + + if (response.OriginalException.InnerException == null) + { + return response.OriginalException.Message; + } + + return response.OriginalException.InnerException.Message; + } + + return response.ServerError.ToString(); } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs index 89b3368..1b2cfec 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchClientProvider.cs @@ -21,7 +21,7 @@ public ElasticsearchClientProvider(IOptions options) var uris = options.Value.Uris.ConvertAll(x => new Uri(x)); var connectionPool = new StaticConnectionPool(uris); var settings = new ConnectionSettings(connectionPool); - // .DisableDirectStreaming(); + // .DisableDirectStreaming() // .OnRequestCompleted(callDetails => // { // // Print Request DSL diff --git a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs index 73955e8..fa16054 100644 --- a/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs +++ b/src/AElf.EntityMapping.Elasticsearch/IElasticsearchQueryableFactory.cs @@ -1,4 +1,6 @@ using AElf.EntityMapping.Elasticsearch.Linq; +using AElf.EntityMapping.Elasticsearch.Options; +using Microsoft.Extensions.Options; using Nest; using Volo.Abp.Domain.Entities; @@ -14,14 +16,18 @@ public class ElasticsearchQueryableFactory : IElasticsearchQueryableFac where TEntity : class, IEntity { private readonly ICollectionNameProvider _collectionNameProvider; + private readonly ElasticsearchOptions _elasticsearchOptions; - public ElasticsearchQueryableFactory(ICollectionNameProvider collectionNameProvider) + public ElasticsearchQueryableFactory(ICollectionNameProvider collectionNameProvider, + IOptions elasticsearchOptions) { _collectionNameProvider = collectionNameProvider; + _elasticsearchOptions = elasticsearchOptions.Value; } - public ElasticsearchQueryable Create(IElasticClient client, string index = null) + public ElasticsearchQueryable Create(IElasticClient client, + string index = null) { - return new ElasticsearchQueryable(client, _collectionNameProvider, index); + return new ElasticsearchQueryable(client, _collectionNameProvider, index, _elasticsearchOptions); } } \ No newline at end of file diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs index b856206..77ecbe7 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticGeneratorQueryModelVisitor.cs @@ -1,6 +1,9 @@ using System.Collections.ObjectModel; using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Linq; +using AElf.EntityMapping.Options; +using Microsoft.Extensions.Options; using Nest; using Remotion.Linq; using Remotion.Linq.Clauses; @@ -13,12 +16,15 @@ public class ElasticsearchGeneratorQueryModelVisitor : QueryModelVisitorBase { private readonly PropertyNameInferrerParser _propertyNameInferrerParser; private readonly INodeVisitor _nodeVisitor; + private readonly ElasticsearchOptions _elasticsearchOptions; private QueryAggregator QueryAggregator { get; set; } = new QueryAggregator(); - public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser) + public ElasticsearchGeneratorQueryModelVisitor(PropertyNameInferrerParser propertyNameInferrerParser, + ElasticsearchOptions elasticsearchOptions) { _propertyNameInferrerParser = propertyNameInferrerParser; _nodeVisitor = new NodeVisitor(); + _elasticsearchOptions = elasticsearchOptions; } public QueryAggregator GenerateElasticQuery(QueryModel queryModel) @@ -48,7 +54,7 @@ public override void VisitMainFromClause(MainFromClause fromClause, QueryModel q public override void VisitWhereClause(WhereClause whereClause, QueryModel queryModel, int index) { - var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser); + var tree = new GeneratorExpressionTreeVisitor(_propertyNameInferrerParser, _elasticsearchOptions); tree.Visit(whereClause.Predicate); if (QueryAggregator.Query == null) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs index efd629c..1d5b521 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryExecutor.cs @@ -3,6 +3,7 @@ using System.Dynamic; using System.Linq.Expressions; using AElf.EntityMapping.Elasticsearch.Exceptions; +using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Nest; using Newtonsoft.Json; @@ -23,16 +24,20 @@ public class ElasticsearchQueryExecutor: IQueryExecutor private readonly JsonSerializerSettings _deserializerSettings; private readonly ICollectionNameProvider _collectionNameProvider; private const int ElasticQueryLimit = 10000; + private readonly ElasticsearchOptions _elasticsearchOptions; public ElasticsearchQueryExecutor(IElasticClient elasticClient, - ICollectionNameProvider collectionNameProvider, string index) + ICollectionNameProvider collectionNameProvider, string index, + ElasticsearchOptions elasticsearchOptions) { _elasticClient = elasticClient; _collectionNameProvider = collectionNameProvider; _index = index; _propertyNameInferrerParser = new PropertyNameInferrerParser(_elasticClient); + _elasticsearchOptions = elasticsearchOptions; _elasticsearchGeneratorQueryModelVisitor = - new ElasticsearchGeneratorQueryModelVisitor(_propertyNameInferrerParser); + new ElasticsearchGeneratorQueryModelVisitor(_propertyNameInferrerParser, + _elasticsearchOptions); _deserializerSettings = new JsonSerializerSettings { // Nest maps TimeSpan as a long (TimeSpan ticks) diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs index 074c5bd..6e0cf25 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/ElasticsearchQueryable.cs @@ -1,4 +1,5 @@ using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using AElf.EntityMapping.Linq; using Nest; using Remotion.Linq; @@ -10,10 +11,10 @@ public class ElasticsearchQueryable : QueryableBase, IElasticsearchQueryab where T : class, IEntity { public ElasticsearchQueryable(IElasticClient elasticClient, ICollectionNameProvider collectionNameProvider, - string index) - : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), + string index, ElasticsearchOptions elasticsearchOptions) + : base(new DefaultQueryProvider(typeof(ElasticsearchQueryable<>), QueryParserFactory.Create(), - new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index))) + new ElasticsearchQueryExecutor(elasticClient, collectionNameProvider, index, elasticsearchOptions))) { } diff --git a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs index 7aa1750..9c71ce6 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Linq/GeneratorExpressionTreeVisitor.cs @@ -1,4 +1,5 @@ using System.Linq.Expressions; +using AElf.EntityMapping.Elasticsearch.Options; using Elasticsearch.Net; using Remotion.Linq.Clauses; using Remotion.Linq.Clauses.Expressions; @@ -10,6 +11,7 @@ namespace AElf.EntityMapping.Elasticsearch.Linq public class GeneratorExpressionTreeVisitor : ThrowingExpressionVisitor { private readonly PropertyNameInferrerParser _propertyNameInferrerParser; + private readonly ElasticsearchOptions _elasticsearchOptions; private object Value { get; set; } private string PropertyName { get; set; } @@ -19,9 +21,11 @@ public class GeneratorExpressionTreeVisitor : ThrowingExpressionVisitor public IDictionary QueryMap { get; } = new Dictionary(); - public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser) + public GeneratorExpressionTreeVisitor(PropertyNameInferrerParser propertyNameInferrerParser, + ElasticsearchOptions elasticsearchOptions) { _propertyNameInferrerParser = propertyNameInferrerParser; + _elasticsearchOptions = elasticsearchOptions; } protected override Expression VisitUnary(UnaryExpression expression) @@ -129,22 +133,22 @@ protected override Expression VisitMethodCall(MethodCallExpression expression) // private string GetFullNameKey(MemberExpression memberExpression) // { - // var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); - // while (memberExpression.Expression != null) - // { - // memberExpression = memberExpression.Expression as MemberExpression; - // if (memberExpression == null) - // { - // break; - // } - // - // key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); - // return key; - // } - // - // return key; + // var key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name); + // while (memberExpression.Expression != null) + // { + // memberExpression = memberExpression.Expression as MemberExpression; + // if (memberExpression == null) + // { + // break; + // } + // + // key = _propertyNameInferrerParser.Parser(memberExpression.Member.Name + "." + key); + // return key; // } - + // + // return key; + // } + private string GetFullPropertyPath(Expression expression) { switch (expression) @@ -162,6 +166,7 @@ private string GetFullPropertyPath(Expression expression) var collectionPath = GetFullPropertyPath(methodCallExpression.Object); return collectionPath; // Returns the path of the collection directly, without adding an index } + break; } @@ -203,15 +208,15 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) Visit(containsResultOperator.Item); Visit(expression.QueryModel.MainFromClause.FromExpression); - if (containsResultOperator.Item.Type == typeof(Guid)) + //Check if the number of items in the Terms query array within the Contains clause is too large. + if (expression.QueryModel.MainFromClause + .FromExpression is ConstantExpression constantExpression) { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); + CheckTermsArrayLength(constantExpression); } - if (containsResultOperator.Item.Type == typeof(Guid?)) - { - query = new TermsNode(PropertyName, ((IEnumerable)Value).Select(x => x.ToString())); - } + // Handling different types + query = GetDifferentTypesTermsQueryNode(); QueryMap[expression] = ParseQuery(query); break; @@ -227,23 +232,32 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) foreach (var whereClause in whereClauses) { - Visit(whereClause.Predicate); - Node tmp = (Node)QueryMap[whereClause.Predicate].Clone(); - QueryMap[expression] = tmp; - QueryMap[expression].IsSubQuery = true; - QueryMap[expression].SubQueryPath = from; //from.ToLower(); - QueryMap[expression].SubQueryFullPath = fullPath; -// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); - BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; - if (predicate.Left is BinaryExpression) + if (whereClause.Predicate is SubQueryExpression subQueryExpression) { - VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true); + HandleNestedContains(subQueryExpression, expression, from, + fullPath); } - - if (predicate.Right is BinaryExpression) + else { - VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true); + Visit(whereClause.Predicate); + Node tmp = (Node)QueryMap[whereClause.Predicate].Clone(); + QueryMap[expression] = tmp; + QueryMap[expression].IsSubQuery = true; + QueryMap[expression].SubQueryPath = from; //from.ToLower(); + QueryMap[expression].SubQueryFullPath = fullPath; +// VisitBinarySetSubQuery((BinaryExpression)whereClause.Predicate, from, fullPath, true); + BinaryExpression predicate = (BinaryExpression)whereClause.Predicate; + if (predicate.Left is BinaryExpression) + { + VisitBinarySetSubQuery((BinaryExpression)predicate.Left, from, fullPath, true); + } + + if (predicate.Right is BinaryExpression) + { + VisitBinarySetSubQuery((BinaryExpression)predicate.Right, from, fullPath, true); + } } + } break; @@ -281,6 +295,98 @@ protected override Expression VisitSubQuery(SubQueryExpression expression) return expression; } + private void HandleNestedContains(SubQueryExpression subQueryExpression, Expression expression, + string subQueryPath, string subQueryFullPath) + { + if (subQueryExpression == null || expression == null) + throw new ArgumentNullException("SubQueryExpression or expression cannot be null."); + + //Check if the number of items in the Terms query array within the Contains clause is too large. + if (subQueryExpression.QueryModel.MainFromClause + .FromExpression is ConstantExpression constantExpression) + { + CheckTermsArrayLength(constantExpression); + } + + foreach (var resultOperator in subQueryExpression.QueryModel.ResultOperators) + { + switch (resultOperator) + { + case ContainsResultOperator containsResultOperator: + Visit(containsResultOperator.Item); + Visit(subQueryExpression.QueryModel.MainFromClause.FromExpression); + break; + } + } + + Node query; + query = GetDifferentTypesTermsQueryNode(); + + QueryMap[expression] = ParseQuery(query); + QueryMap[expression].IsSubQuery = true; + QueryMap[expression].SubQueryPath = subQueryPath; //from.ToLower(); + QueryMap[expression].SubQueryFullPath = subQueryFullPath; + } + + private Node GetDifferentTypesTermsQueryNode() + { + Node query; + if (PropertyType == typeof(Guid)) + { + query = GetTermsNode(); + } + else if (PropertyType == typeof(int)) + { + query = GetTermsNode(); + } + else if (PropertyType == typeof(long)) + { + query = GetTermsNode(); + } + else if (PropertyType == typeof(double)) + { + query = GetTermsNode(); + } + else if (PropertyType == typeof(bool)) + { + query = GetTermsNode(); + } + else if (PropertyType == typeof(DateTime)) + { + query = new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString("o"))); + } + else if (PropertyType == typeof(string)) + { + query = new TermsNode(PropertyName, (IEnumerable)Value); + } + else + { + throw new NotSupportedException($"Type {PropertyType.Name} is not supported for Terms queries."); + } + + return query; + } + + private TermsNode GetTermsNode() + { + return new TermsNode(PropertyName, + ((IEnumerable)Value).Select(x => x.ToString())); + } + + private void CheckTermsArrayLength(ConstantExpression constantExpression) + { + if (constantExpression.Value is System.Collections.IEnumerable objectList) + { + var count = objectList.Cast().Count(); + if (count > _elasticsearchOptions.TermsArrayMaxLength) + { + throw new ArgumentException( + $"The array input for Terms query is too large, exceeding {_elasticsearchOptions.TermsArrayMaxLength} items."); + } + } + } + protected override Expression VisitQuerySourceReference(QuerySourceReferenceExpression expression) { return expression; @@ -597,7 +703,8 @@ private object ConvertEnumValue(Type entityType, string propertyName, object val return (int)enumValue; } - protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, bool parentIsSubQuery) + protected void VisitBinarySetSubQuery(BinaryExpression expression, string path, string fullPath, + bool parentIsSubQuery) { if (expression.Left is BinaryExpression && expression.Right is ConstantExpression) { diff --git a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs index bf6e534..464d72b 100644 --- a/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs +++ b/src/AElf.EntityMapping.Elasticsearch/Options/ElasticsearchOptions.cs @@ -9,4 +9,5 @@ public class ElasticsearchOptions public int NumberOfReplicas { get; set; } = 1; public Refresh Refresh { get; set; } = Refresh.False; public int MaxResultWindow { get; set; } = 10000; + public int TermsArrayMaxLength { get; set; } = 100; } \ No newline at end of file diff --git a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs index bee149d..98f9b5a 100644 --- a/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs +++ b/test/AElf.EntityMapping.Elasticsearch.Tests/Repositories/ElasticsearchRepositoryTests.cs @@ -593,6 +593,96 @@ public async Task GetList_Nested_Test() filterList.Count.ShouldBe(1); } + [Fact] + public async Task GetList_Terms_Test() + { + var timeNow = DateTime.Now; + for (int i = 1; i <= 7; i++) + { + var blockIndex = new BlockIndex + { + Id = "block" + i, + BlockHash = "BlockHash" + i, + BlockHeight = i, + BlockTime = timeNow.AddDays(i), + LogEventCount = i, + ChainId = "AELF" + }; + await _elasticsearchRepository.AddAsync(blockIndex); + } + + List inputs = new List() + { + "BlockHash2", + "BlockHash3", + "BlockHash4" + }; + + var queryable = await _elasticsearchRepository.GetQueryableAsync(); + + var predicates = inputs + .Select(s => (Expression>)(info => info.BlockHash == s)) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable.Where(predicates).ToList(); + filterList_predicate.Count.ShouldBe(3); + + var filterList = queryable.Where(item => inputs.Contains(item.BlockHash)).ToList(); + filterList.Count.ShouldBe(3); + + List heights = new List() + { + 4, 5 + }; + Expression> mustQuery = item => heights.Contains(item.BlockHeight); + var filterList_heights = queryable.Where(mustQuery).ToList(); + filterList_heights.Count.ShouldBe(2); + + List times = new List() + { + DateTime.Now, timeNow.AddDays(1), timeNow.AddDays(2) + }; + Expression> termsQuery = item => times.Contains(item.BlockTime); + var filterList_times = queryable.Where(termsQuery).ToList(); + filterList_times.Count.ShouldBe(2); + } + + [Fact] + public async Task GetNestedList_Terms_Test() + { + //clear data for unit test + ClearTransactionIndex("AELF", 100, 110); + + Thread.Sleep(2000); + //Unit Test 14 + var transaction_100 = MockNewTransactionEtoData(100, false, "token_contract_address", "DonateResourceToken"); + var transaction_101 = MockNewTransactionEtoData(101, false, "", ""); + var transaction_103 = MockNewTransactionEtoData(103, false, "consensus_contract_address", "UpdateValue"); + var transaction_110 = MockNewTransactionEtoData(110, true, "consensus_contract_address", "UpdateTinyBlockInformation"); + await _transactionIndexRepository.AddAsync(transaction_100); + await _transactionIndexRepository.AddAsync(transaction_101); + await _transactionIndexRepository.AddAsync(transaction_103); + await _transactionIndexRepository.AddAsync(transaction_110); + + List inputs = new List() + { + 101, + 103 + }; + var queryable_predicate = await _transactionIndexRepository.GetQueryableAsync(); + var predicates = inputs + .Select(s => (Expression>)(info => info.LogEvents.Any(x => x.BlockHeight == s))) + .Aggregate((prev, next) => prev.Or(next)); + var filterList_predicate = queryable_predicate.Where(predicates).ToList(); + filterList_predicate.Count.ShouldBe(2); + + Expression> mustQuery = item => + item.LogEvents.Any(x => inputs.Contains(x.BlockHeight)); + + var queryable = await _transactionIndexRepository.GetQueryableAsync(); + var filterList = queryable.Where(mustQuery).ToList(); + filterList.Count.ShouldBe(2); + } + [Fact] public async Task SubObjectQueryTest() {